{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "03156b44",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:15:21.274796Z",
     "iopub.status.busy": "2023-12-18T20:15:21.274375Z",
     "iopub.status.idle": "2023-12-18T20:15:36.495008Z",
     "shell.execute_reply": "2023-12-18T20:15:36.493974Z"
    },
    "papermill": {
     "duration": 15.233337,
     "end_time": "2023-12-18T20:15:36.497595",
     "exception": false,
     "start_time": "2023-12-18T20:15:21.264258",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# NS\n",
    "#optiver-ver (for ensemble)\n",
    "\n",
    "#optiver-ver-nn-modeling/inference-v1\n",
    "#optiver-ver-rnn-modeling/inference-v1\n",
    "#optiver-ver-lgb-modeling/inference-v1\n",
    "\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "import lightgbm as lgb\n",
    "import xgboost as xgb\n",
    "from sklearn.linear_model import Ridge\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.stats import hmean\n",
    "\n",
    "from sklearn.ensemble import HistGradientBoostingRegressor\n",
    "import itertools\n",
    "import pickle\n",
    "import joblib\n",
    "from itertools import combinations\n",
    "from tqdm import tqdm\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.layers import Input, Dense, Embedding, Flatten, Concatenate, GaussianNoise\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.experimental import CosineDecay\n",
    "from tensorflow.keras.optimizers.schedules import ExponentialDecay\n",
    "from tensorflow.keras.layers import concatenate,Dropout\n",
    "import pickle\n",
    "from tensorflow.keras.models import load_model\n",
    "import os \n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.losses import Huber\n",
    "from tensorflow.keras.metrics import MeanAbsoluteError\n",
    "from tensorflow.keras.callbacks import Callback\n",
    "import random\n",
    "from tensorflow.keras.layers import Input, Embedding, Lambda, Reshape, LSTM, Dense, BatchNormalization, Dropout, concatenate\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.layers import ZeroPadding1D\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import RepeatVector\n",
    "\n",
    "from tensorflow.keras.layers import Input, Embedding, Lambda, Reshape, LSTM, Dense, BatchNormalization, Dropout, concatenate\n",
    "from tensorflow.keras import backend as K\n",
    "from tensorflow.keras.layers import ZeroPadding1D, Activation\n",
    "from tensorflow.keras.layers import Conv1D\n",
    "from tensorflow.keras.layers import RepeatVector\n",
    "from tensorflow.keras.layers import MaxPooling1D, AveragePooling1D\n",
    "from tensorflow.keras.layers import Add\n",
    "\n",
    "#---------------------------------------------------------------------- setup dashboard ------------------------------------------------------------\n",
    "\n",
    "kaggle              = True\n",
    "is_inference        = True\n",
    "load_models         = True\n",
    "run_pipeline        = False\n",
    "train_models        = False\n",
    "is_lgb              = True    #1\n",
    "is_nn               = True    #2\n",
    "is_rnn              = True   #3\n",
    "simulation          = False\n",
    "online_learning     = True\n",
    "\n",
    "manage_memory       = True\n",
    "memory_threshold    = 24576  #24GB\n",
    "\n",
    "ens_models          = [0.5,0.3,0.2]\n",
    "\n",
    "is_lgb_online              = True    #1\n",
    "is_nn_online               = True    #2\n",
    "is_rnn_online              = False   #3\n",
    "\n",
    "#public-validation\n",
    "# dates_train = [0,390]\n",
    "# dates_test = [391,480]\n",
    "\n",
    "#full-inference\n",
    "dates_train = [0,480]\n",
    "dates_test = [-1,-1]\n",
    "\n",
    "num_models ={'lgb':1,'nn':1,'rnn':1} \n",
    "\n",
    "\n",
    "if kaggle:\n",
    "    train_path = \"/kaggle/input/optiver-trading-at-the-close/train.csv\"\n",
    "    models_path = \"/kaggle/input/optiver-258-models/\"\n",
    "else:\n",
    "    models_path = \"optiver-inference-models/\"\n",
    "    train_path = \"train.csv\"\n",
    "\n",
    "if dates_train[1]!=480:\n",
    "    models_path = \"vals/\"\n",
    "#---------------------------------------------------------------------- setup dashboard ------------------------------------------------------------\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "244805cb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:15:36.515212Z",
     "iopub.status.busy": "2023-12-18T20:15:36.514447Z",
     "iopub.status.idle": "2023-12-18T20:15:36.520855Z",
     "shell.execute_reply": "2023-12-18T20:15:36.519838Z"
    },
    "papermill": {
     "duration": 0.017455,
     "end_time": "2023-12-18T20:15:36.522987",
     "exception": false,
     "start_time": "2023-12-18T20:15:36.505532",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title params dashboard\n",
    "lgb_params = {\n",
    "            'learning_rate'     : 0.05,  #0.005,0.05\n",
    "            'max_depth'         : 14, #14\n",
    "            'n_estimators'      : 5000,\n",
    "            'num_leaves'        : 31,    #511,31,1023\n",
    "            'objective'         : 'mae',\n",
    "            'subsample'         : .2, \n",
    "            'colsample_bytree'  : .3,\n",
    "            'num_threads'       : 32,\n",
    "            'device'            : 'gpu',\n",
    "            # 'reg_alpha'         : 0.1,\n",
    "            # 'reg_lambda'        : 4,\n",
    "        }\n",
    "\n",
    "nn_ep          = 60\n",
    "nn_lr          = 0.001\n",
    "nn_bs          = 2**15\n",
    "\n",
    "\n",
    "rnn_ep         = 60\n",
    "rnn_lr         = 0.001\n",
    "rnn_bs         = 2**12\n",
    "window_size    = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4c97c65e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:15:36.539792Z",
     "iopub.status.busy": "2023-12-18T20:15:36.539378Z",
     "iopub.status.idle": "2023-12-18T20:15:54.973899Z",
     "shell.execute_reply": "2023-12-18T20:15:54.972526Z"
    },
    "papermill": {
     "duration": 18.446184,
     "end_time": "2023-12-18T20:15:54.976751",
     "exception": false,
     "start_time": "2023-12-18T20:15:36.530567",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The 'target' column has 88 NaN values.\n",
      "converting prices columns to float32 values.\n"
     ]
    }
   ],
   "source": [
    "#@title read train\n",
    "\n",
    "colab = False\n",
    "if colab:\n",
    "  from google.colab import drive\n",
    "  drive.mount('/content/drive')\n",
    "  train_path = '/content/drive/My Drive/optiver/train.csv'\n",
    "  models_path = \"/content/drive/My Drive/optiver/\"\n",
    "else:\n",
    "    if not kaggle:\n",
    "        from public_timeseries_testing_util import MockApi\n",
    "\n",
    "pd.set_option('mode.chained_assignment', None)\n",
    "\n",
    "\n",
    "def convert_price_cols_float32(df):\n",
    "\n",
    "    # Columns containing 'price'\n",
    "    price_columns = [col for col in df.columns if 'price' in col]\n",
    "    df[price_columns] = df[price_columns].astype('float32')\n",
    "\n",
    "    # Columns containing 'wap'\n",
    "    wap_columns = [col for col in df.columns if 'wap' in col]\n",
    "    df[wap_columns] = df[wap_columns].astype('float32')\n",
    "\n",
    "    return df\n",
    "\n",
    "train = pd.read_csv(train_path).drop(['row_id', 'time_id'], axis = 1)\n",
    "nan_count = train['target'].isna().sum()\n",
    "print(f\"The 'target' column has {nan_count} NaN values.\")\n",
    "\n",
    "target_median = train['target'].median()\n",
    "train['target'].fillna(target_median, inplace=True)\n",
    "\n",
    "print(f\"converting prices columns to float32 values.\")\n",
    "train = convert_price_cols_float32(train)\n",
    "# ----------------------------- Reading train data -------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "045148b0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:15:54.994766Z",
     "iopub.status.busy": "2023-12-18T20:15:54.994363Z",
     "iopub.status.idle": "2023-12-18T20:15:55.016242Z",
     "shell.execute_reply": "2023-12-18T20:15:55.015082Z"
    },
    "papermill": {
     "duration": 0.033586,
     "end_time": "2023-12-18T20:15:55.018673",
     "exception": false,
     "start_time": "2023-12-18T20:15:54.985087",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title functions\n",
    "\n",
    "def split_by_date(df, dates):\n",
    "\n",
    "    df_start, df_end = dates\n",
    "    df = df[(df['date_id'] >= df_start) & (df['date_id'] <=df_end)].reset_index(drop=True)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "def lag_function(df, columns_to_lag, numbers_of_days_to_lag):\n",
    "\n",
    "    df_indexed = df.set_index(['stock_id', 'seconds_in_bucket', 'date_id'])\n",
    "    \n",
    "    for column_to_lag in columns_to_lag:\n",
    "        for number_days_to_lag in numbers_of_days_to_lag:\n",
    "            df_indexed[f'lag{number_days_to_lag}_{column_to_lag}'] = df_indexed.groupby(level=['stock_id', 'seconds_in_bucket'])[column_to_lag].shift(number_days_to_lag)\n",
    "    \n",
    "    df_indexed.reset_index(inplace=True)\n",
    "    \n",
    "    return df_indexed\n",
    "\n",
    "\n",
    "\n",
    "def create_diff_lagged_features_within_date_revised(df, columns_to_lag, numbers_of_lag):\n",
    "    df_copy = df.copy()\n",
    "\n",
    "    # Store the new columns in a list\n",
    "    new_columns = []\n",
    "\n",
    "    # Iterate through each specified lag\n",
    "    for lag in numbers_of_lag:\n",
    "        # Create lagged dataframe once per lag value\n",
    "        lagged_df = df.groupby(['stock_id', 'date_id'])[columns_to_lag].shift(periods=lag)\n",
    "        \n",
    "        # Iterate through each specified column\n",
    "        for column in columns_to_lag:\n",
    "            # Compute the new column\n",
    "            new_col_name = f'{column}_diff_lag{lag}'\n",
    "            new_column = df[column] - lagged_df[column]\n",
    "            \n",
    "            # Store the new column in the list\n",
    "            new_columns.append(new_column.rename(new_col_name))\n",
    "\n",
    "    # Concatenate the original dataframe with the new columns\n",
    "    result_df = pd.concat([df_copy] + new_columns, axis=1)\n",
    "    \n",
    "    return result_df\n",
    "\n",
    "\n",
    "def create_features_to_start_optimized(df, features_list):\n",
    "\n",
    "    first_values_df = df.groupby(['stock_id', 'date_id'])[features_list].transform('first')\n",
    "\n",
    "    for feature in features_list:\n",
    "        feature_to_start_col_name = f'{feature}_to_start'\n",
    "        df[feature_to_start_col_name] = df[feature] - first_values_df[feature]\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def compute_imbalances(df_, columns, prefix = ''):\n",
    "    \"\"\"Computes the differences and imbalances for pairs of columns and stores them in the DataFrame.\"\"\"\n",
    "    df = df_.copy()\n",
    "    for col1, col2 in combinations(columns, 2):\n",
    "        \n",
    "        # Sort the columns lexicographically to ensure consistent ordering\n",
    "        col1, col2 = sorted([col1, col2])\n",
    "        \n",
    "        # Compute imbalance directly without creating a temporary difference column\n",
    "        total = df[col1] + df[col2]\n",
    "        imbalance_column_name = f'{col1}_{col2}_imb{prefix}'\n",
    "        \n",
    "        # Ensure we don't divide by zero\n",
    "        df[imbalance_column_name] = (df[col1] - df[col2]).divide(total, fill_value=np.nan)\n",
    "\n",
    "    return df\n",
    "\n",
    "def compute_percentage_difference(df, columns, prefix = ''):\n",
    "\n",
    "    df_copy = df.copy()\n",
    "    \n",
    "    # Iterate over all combinations of two different price columns\n",
    "    for col1, col2 in combinations(columns, 2):\n",
    "        # Sort the columns lexicographically to ensure consistent ordering\n",
    "        col1, col2 = sorted([col1, col2])\n",
    "\n",
    "        # Create a new column name based on the price columns\n",
    "        new_col_name = f'pct_diff_{col1}_vs_{col2}_{prefix}'\n",
    "\n",
    "        # Compute the percentage difference\n",
    "        df_copy[new_col_name] = (df_copy[col1] - df_copy[col2]) / df_copy[col2] * 100\n",
    "\n",
    "    return df_copy\n",
    "\n",
    "\n",
    "\n",
    "def create_deviation_within_seconds(df, num_features):\n",
    "    groupby_cols = ['date_id', 'seconds_in_bucket']\n",
    "    new_columns = {}  # Dictionary to hold new columns\n",
    "\n",
    "    for feature in num_features:\n",
    "        grouped_median = df.groupby(groupby_cols)[feature].transform('median')\n",
    "        deviation_col_name = f'deviation_from_median_{feature}'\n",
    "        new_columns[deviation_col_name] = df[feature] - grouped_median\n",
    "\n",
    "    # Concatenate all new columns at once\n",
    "    df = pd.concat([df, pd.DataFrame(new_columns)], axis=1)\n",
    "    return df\n",
    "\n",
    "\n",
    "def create_cumsum_features(df, columns_to_compute):\n",
    "    df_copy = df.copy()\n",
    "    \n",
    "    # Group by 'stock_id' and 'date_id' for cumulative sum calculation\n",
    "    grouped = df_copy.groupby(['stock_id', 'date_id'])\n",
    "    \n",
    "    # Calculate cumulative sum for each column within each group\n",
    "    for column in columns_to_compute:\n",
    "        cumsum_col_name = f'{column}_cumsum'\n",
    "        df_copy[cumsum_col_name] = grouped[column].cumsum()\n",
    "    return df_copy\n",
    "\n",
    "\n",
    "def save_pickle(data, file_path):\n",
    " \n",
    "    # Create the directory if it doesn't exist\n",
    "    directory = os.path.dirname(file_path)\n",
    "    if not os.path.exists(directory):\n",
    "        os.makedirs(directory)\n",
    "\n",
    "    # Save the pickle file\n",
    "    with open(file_path, 'wb') as file:\n",
    "        pickle.dump(data, file)\n",
    "\n",
    "    print(f\"Data saved to {file_path}\")\n",
    "    #example: save_pickle(all_data, 'k8/all_data.pkl')\n",
    "\n",
    "def load_pickle(file_path):\n",
    "\n",
    "    # Load and return the data from the pickle file\n",
    "    if os.path.exists(file_path):\n",
    "        with open(file_path, 'rb') as file:\n",
    "            data = pickle.load(file)\n",
    "        return data\n",
    "    else:\n",
    "        raise FileNotFoundError(f\"No such file: {file_path}\")\n",
    "\n",
    "    #example: loaded_data = load_pickle('k8/all_data.pkl')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5f273986",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:15:55.036243Z",
     "iopub.status.busy": "2023-12-18T20:15:55.035878Z",
     "iopub.status.idle": "2023-12-18T20:16:00.958991Z",
     "shell.execute_reply": "2023-12-18T20:16:00.957882Z"
    },
    "papermill": {
     "duration": 5.934976,
     "end_time": "2023-12-18T20:16:00.961532",
     "exception": false,
     "start_time": "2023-12-18T20:15:55.026556",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title global\n",
    "#---------------------------------- Global based on stock_id --------------------------------------\n",
    "\n",
    "def aggregated_features_dic(df):\n",
    "\n",
    "\n",
    "    global_feats = {}\n",
    "\n",
    "    columns_to_aggregate= ['bid_size','ask_size']\n",
    "    groupby_cols=['stock_id']\n",
    "\n",
    "    def q25(x):\n",
    "        return x.quantile(0.25)\n",
    "\n",
    "    def q75(x):\n",
    "        return x.quantile(0.75)\n",
    "\n",
    "    # Define the aggregations\n",
    "    aggregations = ['mean', 'median', 'std', 'min', 'max', q25, q75]\n",
    "\n",
    "    # Prepare a dictionary to hold the aggregated Series\n",
    "\n",
    "\n",
    "    # Perform aggregation for each column and operation\n",
    "    for column in columns_to_aggregate:\n",
    "        for agg in aggregations:\n",
    "            # Define the aggregation function name\n",
    "            if callable(agg):\n",
    "                func_name = agg.__name__\n",
    "            else:\n",
    "                func_name = agg\n",
    "            \n",
    "            # Perform the aggregation\n",
    "            agg_series = df.groupby(groupby_cols)[column].agg(agg)\n",
    "            # Create a new feature name and add it to the dictionary\n",
    "            new_feature_name = f\"{func_name}_{column}\"\n",
    "            global_feats[new_feature_name] = agg_series\n",
    "\n",
    "    global_feats[\"median_size\"] = df.groupby(\"stock_id\")[\"bid_size\"].median() + df.groupby(\"stock_id\")[\"ask_size\"].median()\n",
    "    global_feats[\"std_size\"] = df.groupby(\"stock_id\")[\"bid_size\"].std() + df.groupby(\"stock_id\")[\"ask_size\"].std()\n",
    "    global_feats[\"ptp_size\"] = df.groupby(\"stock_id\")[\"bid_size\"].max() - df.groupby(\"stock_id\")[\"bid_size\"].min()\n",
    "    global_feats[\"median_price\"] = df.groupby(\"stock_id\")[\"bid_price\"].median() + df.groupby(\"stock_id\")[\"ask_price\"].median()\n",
    "    global_feats[\"std_price\"] = df.groupby(\"stock_id\")[\"bid_price\"].std() + df.groupby(\"stock_id\")[\"ask_price\"].std()\n",
    "    global_feats[\"ptp_price\"] = df.groupby(\"stock_id\")[\"bid_price\"].max() - df.groupby(\"stock_id\")[\"ask_price\"].min()\n",
    "\n",
    "\n",
    "    return global_feats\n",
    "\n",
    "aggregated_dic = aggregated_features_dic(train)\n",
    "\n",
    "def map_global(df,dict):\n",
    "    df_ = df.copy()\n",
    "    for key, value in dict.items():\n",
    "        df_[f\"global_{key}\"] = df_[\"stock_id\"].map(value.to_dict())\n",
    "    \n",
    "    return df_\n",
    "#---------------------------------- Global based on stock_id --------------------------------------\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7422c05d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:00.978779Z",
     "iopub.status.busy": "2023-12-18T20:16:00.978346Z",
     "iopub.status.idle": "2023-12-18T20:16:00.988803Z",
     "shell.execute_reply": "2023-12-18T20:16:00.987974Z"
    },
    "papermill": {
     "duration": 0.021287,
     "end_time": "2023-12-18T20:16:00.990823",
     "exception": false,
     "start_time": "2023-12-18T20:16:00.969536",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title helper functions\n",
    "\n",
    "\n",
    "def flatten_outliers_y_train(y_train, lower_quantile=0.01, upper_quantile=0.99):\n",
    "\n",
    "    lower_bound = np.quantile(y_train, lower_quantile)\n",
    "    upper_bound = np.quantile(y_train, upper_quantile)\n",
    "\n",
    "    # Cap values below the lower bound and above the upper bound\n",
    "    y_train_flattened = np.clip(y_train, lower_bound, upper_bound)\n",
    "\n",
    "    return y_train_flattened\n",
    "\n",
    "\n",
    "def create_autocorrelation_features(df, columns, lags):\n",
    "\n",
    "    df_copy = df.copy()\n",
    "    \n",
    "    for column in columns:\n",
    "        for lag in lags:\n",
    "            lagged_series = df_copy[column].shift(lag)\n",
    "            df_copy[f'{column}_autocorr_lag{lag}'] = df_copy[column].corrwith(lagged_series)\n",
    "\n",
    "    return df_copy\n",
    "\n",
    "\n",
    "def calculate_stat_lag(df, num_lags):\n",
    "\n",
    "    lags = [f'lag{i}_target' for i in range(1, num_lags + 1)]\n",
    "\n",
    "    df['target_mean'] = df[lags].mean(axis=1)\n",
    "    df['target_std_dev'] = df[lags].std(axis=1)\n",
    "    df['target_variance'] = df[lags].var(axis=1)\n",
    "    df['target_median'] = df[lags].median(axis=1)\n",
    "    df['target_range'] = df[lags].max(axis=1) - df[lags].min(axis=1)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "def calculate_stat(df, cols, prefix='prices'):\n",
    "\n",
    "    df[f'{prefix}_mean'] = df[cols].mean(axis=1)\n",
    "    df[f'{prefix}_std_dev'] = df[cols].std(axis=1)\n",
    "    df[f'{prefix}_variance'] = df[cols].var(axis=1)\n",
    "    df[f'{prefix}_median'] = df[cols].median(axis=1)\n",
    "    df[f'{prefix}_range'] = df[cols].max(axis=1) - df[cols].min(axis=1)\n",
    "\n",
    "    return df\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "0e2a7762",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.009486Z",
     "iopub.status.busy": "2023-12-18T20:16:01.008403Z",
     "iopub.status.idle": "2023-12-18T20:16:01.024314Z",
     "shell.execute_reply": "2023-12-18T20:16:01.023402Z"
    },
    "papermill": {
     "duration": 0.027627,
     "end_time": "2023-12-18T20:16:01.026900",
     "exception": false,
     "start_time": "2023-12-18T20:16:00.999273",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title pipeline\n",
    "\n",
    "raw_cols          = ['imbalance_size','matched_size','bid_size','ask_size','reference_price','far_price','near_price','bid_price','ask_price','wap','imbalance_buy_sell_flag'] \n",
    "\n",
    "columns_prices    = ['reference_price','far_price','near_price','bid_price','ask_price','wap']\n",
    "columns_4prices   = ['reference_price','bid_price','ask_price','wap']\n",
    "\n",
    "columns_sizes     = ['imbalance_size','matched_size','bid_size','ask_size']\n",
    "columns_flag      = ['imbalance_buy_sell_flag'] \n",
    "\n",
    "\n",
    "diff_lags           = [1, 2, 3, 6, 12, 18, 24] \n",
    "diff_lags_extra     = [30, 36, 42, 48]\n",
    "\n",
    "num_of_target_lags  = 12\n",
    "target_lags         = list(range(1,num_of_target_lags+1))\n",
    "\n",
    "\n",
    "def feature_engineering(df):\n",
    "    \n",
    "    df = df.copy()\n",
    "\n",
    "    df['spread_eng']                            = df['ask_price'] - df['bid_price'] \n",
    "    df['volume_eng']                            = df['bid_size'] + df['ask_size']  \n",
    "    df['volumne_imbalance_eng']                 = df['bid_size'] - df['ask_size']  \n",
    "    \n",
    "    df['imbalance_ratio']                       = df['imbalance_size'] / df['matched_size']  #(RM) Good\n",
    "\n",
    "    df['price_spread_near_far']                 = df['near_price'] - df['far_price']   #RM (debatable)\n",
    "    df['price_wap_difference_eng']              = df['reference_price'] - df['wap']    \n",
    "\n",
    "    df['weighted_imbalance_eng']                = df['imbalance_size'] * df['imbalance_buy_sell_flag'] #very important\n",
    "\n",
    "\n",
    "    df['bid_ask_ratio']                         = df['bid_size'] / df['ask_size'] #(RM) neutral\n",
    "    df['imbalance_to_bid_ratio_eng']            = df['imbalance_size'] / df['bid_size']\n",
    "    df['imbalance_to_ask_ratio_eng']            = df['imbalance_size'] / df['ask_size']\n",
    "    df['matched_size_to_total_size_ratio_eng']  = df['matched_size'] / (df['bid_size'] + df['ask_size'])\n",
    "\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "def feature_pipeline(df):\n",
    "    \n",
    "    if df.empty:\n",
    "        return pd.DataFrame()\n",
    "        \n",
    "    #--------------- connected ------------\n",
    "    df = feature_engineering(df)\n",
    "    df = compute_imbalances(df, columns_sizes,prefix='_sz_')\n",
    "    df = compute_imbalances(df, columns_prices,prefix = '_pr_')\n",
    "\n",
    "\n",
    "\n",
    "    eng_features       = [feature for feature in df.columns if \"_eng\" in feature]\n",
    "    imb_features_all   = [feature for feature in df.columns if \"_imb_\" in feature]\n",
    "    imb_features_price = [feature for feature in df.columns if \"_pr_\" in feature]\n",
    "    imb_features_size  = [feature for feature in df.columns if \"_sz_\" in feature]\n",
    "\n",
    "    #--------------- connected ------------\n",
    "\n",
    "\n",
    "    diff_lag_cols = raw_cols + eng_features\n",
    "    print(f\"diff lagging {len(diff_lag_cols)} columns for {len(diff_lags)} lags.\")\n",
    "    df = create_diff_lagged_features_within_date_revised(df,diff_lag_cols,diff_lags)\n",
    "\n",
    "    cumsum_columns = columns_sizes + imb_features_size + eng_features \n",
    "    print(f\"cumsum for {len(cumsum_columns)} cols.\")\n",
    "    df = create_cumsum_features(df, cumsum_columns)\n",
    "\n",
    "    deviation_cols = raw_cols + eng_features + imb_features_size #+ imb_features_price\n",
    "    print(f\"deviation {len(deviation_cols)} columns within seconds.\")\n",
    "    df = create_deviation_within_seconds(df,deviation_cols)\n",
    "\n",
    "    print(f\"lagging target column for {len(target_lags)} lags.\")\n",
    "    df = lag_function(df, ['target'], target_lags)\n",
    "\n",
    "    df = map_global(df,aggregated_dic)\n",
    "\n",
    "    df = calculate_stat_lag(df, num_lags=num_of_target_lags)\n",
    "\n",
    "    df.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "\n",
    "    print(\"Done...\")\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "765ce252",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.044962Z",
     "iopub.status.busy": "2023-12-18T20:16:01.044600Z",
     "iopub.status.idle": "2023-12-18T20:16:01.056602Z",
     "shell.execute_reply": "2023-12-18T20:16:01.055679Z"
    },
    "papermill": {
     "duration": 0.023792,
     "end_time": "2023-12-18T20:16:01.058638",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.034846",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title NN second pass\n",
    "#------------------------------------------------- NN Second Pass Functions -----------------------------------------------\n",
    "\n",
    "def make_predictions(models, X_test,model = 'nn'):\n",
    "    if model == 'nn':\n",
    "        all_predictions = [model.predict(X_test, batch_size=16384) for model in models]\n",
    "    if model == 'lgb' or model == 'xgb' or model == 'cat':\n",
    "        all_predictions = [model.predict(X_test) for model in models]\n",
    "    prediction = np.mean(all_predictions, axis=0)\n",
    "    return prediction\n",
    "\n",
    "def set_all_seeds(seed):\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    tf.random.set_seed(seed)\n",
    "\n",
    "class BestScoresCallback(Callback):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.best_train_loss = float('inf')\n",
    "        self.best_val_loss = float('inf')\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        train_loss = logs.get('loss', float('inf'))\n",
    "        val_loss = logs.get('val_loss', float('inf'))\n",
    "\n",
    "        if train_loss < self.best_train_loss:\n",
    "            self.best_train_loss = train_loss\n",
    "        if val_loss < self.best_val_loss:\n",
    "            self.best_val_loss = val_loss\n",
    "\n",
    "    def on_train_end(self, logs=None):\n",
    "        print(f\"Best training loss: {self.best_train_loss}, Best validation loss: {self.best_val_loss}\")\n",
    "\n",
    "\n",
    "def second_pass_for_nn(df, numerical_features, categorical_features, is_inference=False):\n",
    "    # Check if the DataFrame is empty\n",
    "    global scaler,medians\n",
    "    \n",
    "    if df.empty:\n",
    "        return None, None\n",
    "\n",
    "    # Work on a copy of the DataFrame to avoid changing the original df\n",
    "    df_copy = df.copy()\n",
    "\n",
    "\n",
    "    # Standard scaling for numerical features\n",
    "    if is_inference:\n",
    "        df_copy.fillna(medians, inplace=True)\n",
    "        df_copy[numerical_features] = scaler.transform(df_copy[numerical_features])\n",
    "\n",
    "    # Preprocess Data\n",
    "    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n",
    "    df_copy['imbalance_buy_sell_flag'] += 1 \n",
    "\n",
    "    # Create input vector\n",
    "    X_cat = [df_copy[cat].values.reshape(-1, 1) for cat in categorical_features]\n",
    "    X_num = df_copy[numerical_features].values\n",
    "    \n",
    "\n",
    "    y = df_copy['target'].values\n",
    "        \n",
    "    return [X_num] + X_cat, y\n",
    "\n",
    "#------------------------------------------------- NN Second Pass Functions -----------------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "164752b6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.076003Z",
     "iopub.status.busy": "2023-12-18T20:16:01.075649Z",
     "iopub.status.idle": "2023-12-18T20:16:01.085216Z",
     "shell.execute_reply": "2023-12-18T20:16:01.084103Z"
    },
    "papermill": {
     "duration": 0.021042,
     "end_time": "2023-12-18T20:16:01.087582",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.066540",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title NN model \n",
    "\n",
    "def create_model(categorical_features,numerical_features,initial_learning_rate=0.001):\n",
    "\n",
    "    inputs = []\n",
    "    embeddings = []\n",
    "\n",
    "    categorical_uniques = {}\n",
    "    categorical_uniques['seconds_in_bucket'] = 55\n",
    "\n",
    "    embedding_dim = {}\n",
    "    embedding_dim['seconds_in_bucket'] = 10\n",
    "\n",
    "\n",
    "    input_num = Input(shape=(len(numerical_features),), name=\"numerical_input\")\n",
    "    inputs.append(input_num)\n",
    "\n",
    "    for cat_feature in categorical_features:\n",
    "        vocab_size = categorical_uniques[cat_feature]\n",
    "        input_cat = Input(shape=(1,), name=f\"input_{cat_feature}\")\n",
    "        embedding = Embedding(vocab_size, embedding_dim[cat_feature], input_length=1, name=f\"embedding_{cat_feature}\")(input_cat)\n",
    "        flattened_embedding = Flatten(name=f\"flatten_{cat_feature}\")(embedding)\n",
    "        inputs.append(input_cat)\n",
    "        embeddings.append(flattened_embedding)\n",
    "\n",
    "    dense_output = concatenate(embeddings + [input_num])\n",
    "\n",
    "    dense_sizes = [512, 256, 128, 64, 32]\n",
    "\n",
    "    for size in dense_sizes:\n",
    "        dense_output = Dense(size, activation='swish')(dense_output)\n",
    "        dense_output = BatchNormalization()(dense_output)\n",
    "        dense_output = Dropout(0.4)(dense_output)\n",
    "\n",
    "    output = Dense(1)(dense_output)\n",
    "\n",
    "\n",
    "    lr_schedule = ExponentialDecay(\n",
    "    initial_learning_rate=initial_learning_rate,\n",
    "    decay_steps=3000,           \n",
    "    decay_rate=0.5,\n",
    "    staircase=True)\n",
    "\n",
    "    model = Model(inputs, output)\n",
    "    optimizer = Adam(learning_rate=lr_schedule)\n",
    "\n",
    "    model.compile(optimizer=optimizer, loss = \"mean_absolute_error\")\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cad2e1b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.105656Z",
     "iopub.status.busy": "2023-12-18T20:16:01.105281Z",
     "iopub.status.idle": "2023-12-18T20:16:01.129377Z",
     "shell.execute_reply": "2023-12-18T20:16:01.128312Z"
    },
    "papermill": {
     "duration": 0.036227,
     "end_time": "2023-12-18T20:16:01.131949",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.095722",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title RNN second pass\n",
    "\n",
    "def precompute_sequences(stock_data, window_size, rnn_numerical_features, rnn_categorical_features):\n",
    "    # Convert DataFrame columns to NumPy arrays\n",
    "    stock_data_num = stock_data[rnn_numerical_features].values\n",
    "    stock_data_cat = stock_data[rnn_categorical_features].values\n",
    "\n",
    "    # Pre-compute all sequences\n",
    "    all_sequences_num = [stock_data_num[max(0, i - window_size + 1):i + 1] for i in range(len(stock_data))]\n",
    "    all_sequences_cat = [stock_data_cat[max(0, i - window_size + 1):i + 1] for i in range(len(stock_data))]\n",
    "\n",
    "    # Add padding if necessary\n",
    "    padded_sequences_num = [np.pad(seq, ((window_size - len(seq), 0), (0, 0)), 'constant') for seq in all_sequences_num]\n",
    "    padded_sequences_cat = [np.pad(seq, ((window_size - len(seq), 0), (0, 0)), 'constant') for seq in all_sequences_cat]\n",
    "\n",
    "    # Combine numerical and categorical features\n",
    "    combined_sequences = np.array([np.concatenate([num, cat], axis=-1) \n",
    "                                   for num, cat in zip(padded_sequences_num, padded_sequences_cat)])\n",
    "\n",
    "    # Extract targets\n",
    "    targets = stock_data['target'].values\n",
    "\n",
    "    return combined_sequences, targets\n",
    "\n",
    "def get_sequence(precomputed_data, time_step):\n",
    "    combined_sequences, targets = precomputed_data\n",
    "    return combined_sequences[time_step], targets[time_step]\n",
    "\n",
    "\n",
    "\n",
    "def create_batches(data, window_size, rnn_numerical_features, rnn_categorical_features, max_time_steps=55):\n",
    "    \n",
    "    grouped = data.groupby(['stock_id', 'date_id'])\n",
    "    all_batches = []\n",
    "    all_targets = []\n",
    "\n",
    "    for _, group in tqdm(grouped, desc=\"Processing groups\"):\n",
    "\n",
    "        # Precompute sequences for the current group\n",
    "        precomputed_data = precompute_sequences(group, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "\n",
    "        # Initialize containers for group sequences and targets\n",
    "        group_sequences = []\n",
    "        group_targets = []\n",
    "\n",
    "        # Iterate over the time steps and retrieve precomputed sequences\n",
    "        for time_step in range(max_time_steps):\n",
    "            sequence, target = get_sequence(precomputed_data, time_step)\n",
    "            if sequence.size > 0: \n",
    "                group_sequences.append(sequence)\n",
    "                group_targets.append(target)\n",
    "\n",
    "        # Extend the main batches with the group's sequences and targets\n",
    "        all_batches.extend(group_sequences)\n",
    "        all_targets.extend(group_targets)\n",
    "\n",
    "    return all_batches, all_targets\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def compute_last_sequence(group, window_size, rnn_numerical_features, rnn_categorical_features):\n",
    "    # Convert DataFrame columns to NumPy arrays\n",
    "    stock_data_num      = group[rnn_numerical_features].values\n",
    "    stock_data_cat      = group[rnn_categorical_features].values\n",
    "    stock_data_target   = group['target'].values\n",
    "\n",
    "    # Find the index of the target second\n",
    "    target_index = len(group) - 1\n",
    "\n",
    "    # Extract the sequence for the target index\n",
    "    sequence_num = stock_data_num[max(0, target_index - window_size + 1):target_index + 1]\n",
    "    sequence_cat = stock_data_cat[max(0, target_index - window_size + 1):target_index + 1]\n",
    "\n",
    "    # Add padding if necessary\n",
    "    padded_sequence_num = np.pad(sequence_num, ((window_size - len(sequence_num), 0), (0, 0)), 'constant')\n",
    "    padded_sequence_cat = np.pad(sequence_cat, ((window_size - len(sequence_cat), 0), (0, 0)), 'constant')\n",
    "\n",
    "    # Combine numerical and categorical features\n",
    "    combined_sequence = np.concatenate([padded_sequence_num, padded_sequence_cat], axis=-1)\n",
    "\n",
    "    # Extract target\n",
    "    target = stock_data_target[-1]\n",
    "\n",
    "    return combined_sequence, target\n",
    "\n",
    "\n",
    "def create_last_batches(data, window_size, rnn_numerical_features, rnn_categorical_features):\n",
    "    \n",
    "    grouped = data.groupby(['stock_id'])\n",
    "    all_batches = []\n",
    "    all_targets = []\n",
    "\n",
    "    for _, group in grouped:\n",
    "        # Compute the sequence for the last data point in the current group\n",
    "        last_sequence, last_target = compute_last_sequence(group, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "\n",
    "        # Check if the sequence is valid (i.e., not empty)\n",
    "        if last_sequence.size > 0: \n",
    "            all_batches.append(last_sequence)\n",
    "            all_targets.append(last_target)\n",
    "\n",
    "    return all_batches, all_targets\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def second_pass_for_rnn(df, rnn_numerical_features, rnn_categorical_features, window_size, is_inference=False):\n",
    "    # Check if the DataFrame is empty\n",
    "    global rnn_scaler,rnn_medians\n",
    "    \n",
    "    if df.empty:\n",
    "        return None, None\n",
    "\n",
    "    # Work on a copy of the DataFrame to avoid changing the original df\n",
    "    df_copy = df.copy()\n",
    "\n",
    "    # Standard scaling for numerical features\n",
    "    if is_inference:\n",
    "        df_copy.fillna(rnn_medians, inplace=True)\n",
    "        df_copy[rnn_numerical_features] = rnn_scaler.transform(df_copy[rnn_numerical_features])\n",
    "        \n",
    "    # Preprocess Data\n",
    "    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n",
    "    df_copy['imbalance_buy_sell_flag'] += 1 \n",
    "\n",
    "    if is_inference:\n",
    "        df_copy_batches, df_copy_targets = create_last_batches(df_copy, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "    else:\n",
    "        df_copy_batches, df_copy_targets = create_batches(df_copy, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "\n",
    "    df_copy_batches = np.array(df_copy_batches)\n",
    "    df_copy_targets = np.array(df_copy_targets)\n",
    "\n",
    "\n",
    "    return df_copy_batches, df_copy_targets\n",
    "\n",
    "\n",
    "def online_pass_for_rnn(df, rnn_numerical_features, rnn_categorical_features, window_size):\n",
    "    # Check if the DataFrame is empty\n",
    "    global rnn_scaler,rnn_medians\n",
    "    \n",
    "    if df.empty:\n",
    "        return None, None\n",
    "\n",
    "    # Work on a copy of the DataFrame to avoid changing the original df\n",
    "    df_copy = df.copy()\n",
    "\n",
    "    # Standard scaling for numerical features\n",
    "    df_copy.fillna(rnn_medians, inplace=True)\n",
    "    df_copy[rnn_numerical_features] = rnn_scaler.transform(df_copy[rnn_numerical_features])\n",
    "        \n",
    "    # Preprocess Data\n",
    "    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n",
    "    df_copy['imbalance_buy_sell_flag'] += 1 \n",
    "\n",
    "\n",
    "    grouped = df_copy.groupby(['stock_id'])\n",
    "    all_batches = []\n",
    "    all_targets = []\n",
    "\n",
    "    for _, group in tqdm(grouped, desc=\"Processing groups\"):\n",
    "        # Precompute sequences for the current group\n",
    "        precomputed_data = precompute_sequences(group, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "\n",
    "        # Initialize containers for group sequences and targets\n",
    "        group_sequences = []\n",
    "        group_targets = []\n",
    "\n",
    "        # Iterate over the time steps and retrieve precomputed sequences\n",
    "        for time_step in range(55):\n",
    "            sequence, target = get_sequence(precomputed_data, time_step)\n",
    "            if sequence.size > 0: \n",
    "                group_sequences.append(sequence)\n",
    "                group_targets.append(target)\n",
    "\n",
    "        # Extend the main batches with the group's sequences and targets\n",
    "        all_batches.extend(group_sequences)\n",
    "        all_targets.extend(group_targets)\n",
    "\n",
    "    df_batches = np.array(all_batches)\n",
    "    df_targets = np.array(all_targets)\n",
    "\n",
    "\n",
    "    return df_batches, df_targets\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "129c6a79",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.150025Z",
     "iopub.status.busy": "2023-12-18T20:16:01.149649Z",
     "iopub.status.idle": "2023-12-18T20:16:01.165607Z",
     "shell.execute_reply": "2023-12-18T20:16:01.164829Z"
    },
    "papermill": {
     "duration": 0.027423,
     "end_time": "2023-12-18T20:16:01.167621",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.140198",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title RNN model\n",
    "\n",
    "def apply_conv_layers(input_layer, kernel_sizes, filters=16, do_ratio=0.5):\n",
    "    conv_outputs = []\n",
    "\n",
    "    for kernel_size in kernel_sizes:\n",
    "        conv_layer = Conv1D(filters=filters, kernel_size=kernel_size, activation='relu', padding='same')(input_layer)\n",
    "        conv_layer = BatchNormalization()(conv_layer)\n",
    "        conv_layer = Dropout(do_ratio)(conv_layer)\n",
    "\n",
    "        shortcut = conv_layer\n",
    "\n",
    "        conv_layer = Conv1D(filters=filters, kernel_size=kernel_size, padding='same')(conv_layer)\n",
    "        conv_layer = BatchNormalization()(conv_layer)\n",
    "        conv_layer = Activation('relu')(conv_layer)\n",
    "\n",
    "        # Add the output of the first Conv1D layer\n",
    "        conv_layer = Add()([conv_layer, shortcut])\n",
    "        conv_outputs.append(conv_layer)\n",
    "\n",
    "\n",
    "    concatenated_conv = Concatenate(axis=-1)(conv_outputs)\n",
    "    flattened_conv_output = Flatten()(concatenated_conv)\n",
    "\n",
    "    return flattened_conv_output\n",
    "\n",
    "def create_rnn_model_with_residual(window_size, numerical_features, initial_learning_rate=0.001):\n",
    "\n",
    "    categorical_features = 'seconds_in_bucket'\n",
    "    categorical_uniques  = { 'seconds_in_bucket' : 55}\n",
    "    embedding_dim        = {'seconds_in_bucket' : 10}\n",
    "\n",
    "    input_layer = Input(shape=(window_size, len(numerical_features) + 1), name=\"combined_input\")\n",
    "\n",
    "    # Split the input into numerical and categorical parts\n",
    "    numerical_input = Lambda(lambda x: x[:, :, :-1], name=\"numerical_part\")(input_layer)\n",
    "    categorical_input = Lambda(lambda x: x[:, :, -1:], name=\"categorical_part\")(input_layer)\n",
    "\n",
    "    first_numerical = Lambda(lambda x: x[:, 0])(numerical_input)\n",
    "\n",
    "\n",
    "    # diffrentiate layers\n",
    "    def create_difference_layer(lag):\n",
    "        return Lambda(lambda x: x[:, lag:, :] - x[:, :-lag, :], name=f\"difference_layer_lag{lag}\")\n",
    "\n",
    "    difference_layers = []\n",
    "    for lag in range(1, window_size):\n",
    "        diff_layer = create_difference_layer(lag)(numerical_input)\n",
    "        padding = ZeroPadding1D(padding=(lag, 0))(diff_layer)  # Add padding to the beginning of the sequence\n",
    "        difference_layers.append(padding)\n",
    "    combined_diff_layer = Concatenate(name=\"combined_difference_layer\")(difference_layers)\n",
    "\n",
    "\n",
    "    # Embedding for categorical part\n",
    "    vocab_size, embedding_dim = categorical_uniques[categorical_features], embedding_dim[categorical_features]\n",
    "    embedding = Embedding(vocab_size, embedding_dim, input_length=window_size)(categorical_input)\n",
    "    embedding = Reshape((window_size, -1))(embedding)\n",
    "\n",
    "    first_embedding = Lambda(lambda x: x[:, 0])(embedding)\n",
    "\n",
    "    # Concatenate numerical input and embedding\n",
    "    # conv_input = concatenate([enhanced_numerical_input, embedding], axis=-1)\n",
    "\n",
    "    kernel_sizes = [2,3]\n",
    "    do_ratio = 0.4\n",
    "\n",
    "    flattened_conv_output = apply_conv_layers(numerical_input, kernel_sizes, do_ratio=do_ratio)\n",
    "    flattened_conv_output_cat = apply_conv_layers(embedding, kernel_sizes, do_ratio=do_ratio)\n",
    "    flattened_conv_output_diff = apply_conv_layers(combined_diff_layer, kernel_sizes, do_ratio=do_ratio)\n",
    "\n",
    "\n",
    "    dense_output = Concatenate(axis=-1)([flattened_conv_output,flattened_conv_output_cat,flattened_conv_output_diff, Reshape((-1,))(combined_diff_layer),first_numerical,first_embedding])\n",
    "\n",
    "    dense_sizes = [512, 256, 128, 64, 32]\n",
    "    do_ratio = 0.5\n",
    "    for size in dense_sizes:\n",
    "        dense_output = Dense(size, activation='swish')(dense_output)\n",
    "        dense_output = BatchNormalization()(dense_output)\n",
    "        dense_output = Dropout(do_ratio)(dense_output)\n",
    "\n",
    "    # Output layer\n",
    "    output = Dense(1, name='output_layer')(dense_output)\n",
    "\n",
    "    # Learning rate schedule\n",
    "    lr_schedule = ExponentialDecay(\n",
    "        initial_learning_rate=initial_learning_rate,\n",
    "        decay_steps=10000,\n",
    "        decay_rate=0.7,\n",
    "        staircase=True)\n",
    "\n",
    "    # Create and compile the model\n",
    "    model = Model(inputs=input_layer, outputs=output)\n",
    "    optimizer = Adam(learning_rate=lr_schedule)\n",
    "\n",
    "    model.compile(optimizer=optimizer, loss=\"mean_absolute_error\")\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f193d5ac",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.186300Z",
     "iopub.status.busy": "2023-12-18T20:16:01.185290Z",
     "iopub.status.idle": "2023-12-18T20:16:01.197739Z",
     "shell.execute_reply": "2023-12-18T20:16:01.196756Z"
    },
    "papermill": {
     "duration": 0.024204,
     "end_time": "2023-12-18T20:16:01.200042",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.175838",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title runnig pipeline\n",
    "\n",
    "excluded_columns = ['row_id', 'date_id', 'time_id', 'target','stock_return']  \n",
    "\n",
    "if run_pipeline:\n",
    "\n",
    "    train_eng = feature_pipeline(train)\n",
    "    \n",
    "    \n",
    "    if is_nn:\n",
    "        excluded_columns = excluded_columns + ['stock_id']\n",
    "\n",
    "        features = [col for col in train_eng.columns if col not in excluded_columns]\n",
    "        categorical_features =  ['seconds_in_bucket']\n",
    "        numerical_features = [feat for feat in features if feat not in categorical_features]\n",
    "        print(\"we have {} numerical and {} categorical\".format(len(numerical_features),len(categorical_features)))\n",
    "\n",
    "\n",
    "\n",
    "        scaler = StandardScaler()\n",
    "        medians = train_eng.median()\n",
    "        \n",
    "        train_eng.fillna(medians, inplace=True)\n",
    "        train_eng[numerical_features] = scaler.fit_transform(train_eng[numerical_features])\n",
    "\n",
    "    \n",
    "        all_data = {\n",
    "            \"scaler\": scaler,\n",
    "            \"medians\": medians,\n",
    "            \"categorical_features\": categorical_features,\n",
    "            \"numerical_features\": numerical_features\n",
    "        }\n",
    "\n",
    "        save_pickle(all_data, f'{models_path}all_data.pkl')\n",
    "        print(\"Pipline Done!\")\n",
    "\n",
    "    if is_rnn:\n",
    "        excluded_columns = excluded_columns + ['stock_id']\n",
    "\n",
    "        features = [col for col in train_eng.columns if col not in excluded_columns]\n",
    "        rnn_categorical_features =  ['seconds_in_bucket']\n",
    "        rnn_numerical_features = [feat for feat in features if feat not in rnn_categorical_features]\n",
    "        print(\"we have {} numerical and {} categorical\".format(len(rnn_numerical_features),len(rnn_categorical_features)))\n",
    "\n",
    "\n",
    "        rnn_scaler = StandardScaler()\n",
    "        rnn_medians = train_eng.median()\n",
    "        \n",
    "        train_eng.fillna(rnn_medians, inplace=True)\n",
    "        train_eng[rnn_numerical_features] = rnn_scaler.fit_transform(train_eng[rnn_numerical_features])\n",
    "\n",
    "    \n",
    "        rnn_all_data = {\n",
    "            \"rnn_scaler\": rnn_scaler,\n",
    "            \"rnn_medians\": rnn_medians,\n",
    "            \"rnn_categorical_features\": rnn_categorical_features,\n",
    "            \"rnn_numerical_features\": rnn_numerical_features\n",
    "        }\n",
    "\n",
    "        save_pickle(rnn_all_data, f'{models_path}rnn_all_data.pkl')\n",
    "        print(\"Pipline Done!\")\n",
    "\n",
    "    if is_lgb:\n",
    "        lgb_features = [col for col in train_eng.columns if col not in excluded_columns]\n",
    "        categorical_features = ['seconds_in_bucket']\n",
    "\n",
    "        print(\"we have {} lgb features\".format(len(lgb_features)))\n",
    "        \n",
    "    train_data       = split_by_date(train_eng, dates_train)\n",
    "    test_data        = split_by_date(train_eng, dates_test)\n",
    "    print(\"number of dates in train = {} , number of dates in test {}\".format (train_data['date_id'].nunique(),test_data['date_id'].nunique()))\n",
    "\n",
    "    cleaning = False\n",
    "    if cleaning:\n",
    "        import gc\n",
    "        #del train\n",
    "        del train_eng\n",
    "        gc.collect()\n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "605e68eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.217955Z",
     "iopub.status.busy": "2023-12-18T20:16:01.217547Z",
     "iopub.status.idle": "2023-12-18T20:16:01.222630Z",
     "shell.execute_reply": "2023-12-18T20:16:01.221743Z"
    },
    "papermill": {
     "duration": 0.016405,
     "end_time": "2023-12-18T20:16:01.224498",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.208093",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title TPU\n",
    "try:\n",
    "    # Create a TPUClusterResolver\n",
    "    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n",
    "    # Connect to the TPU cluster\n",
    "    tf.config.experimental_connect_to_cluster(tpu)\n",
    "    # Initialize the TPU system\n",
    "    tf.tpu.experimental.initialize_tpu_system(tpu)\n",
    "    # Create a TPUStrategy for distributed training\n",
    "    tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n",
    "except ValueError:\n",
    "    tpu_strategy = None  # No TPU found\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "3727d326",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.242806Z",
     "iopub.status.busy": "2023-12-18T20:16:01.241869Z",
     "iopub.status.idle": "2023-12-18T20:16:01.249167Z",
     "shell.execute_reply": "2023-12-18T20:16:01.248380Z"
    },
    "papermill": {
     "duration": 0.018555,
     "end_time": "2023-12-18T20:16:01.251115",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.232560",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title prepare train models\n",
    "if train_models:\n",
    "\n",
    "    if test_data.empty:\n",
    "        print(\"--------------test data is empty so adjusting the last date for test-----------------\")\n",
    "        test_data = train_data.query(\"date_id == 480\").copy()\n",
    "\n",
    "    if is_lgb:\n",
    "\n",
    "\n",
    "        X_train, y_train = train_data[lgb_features], train_data['target']\n",
    "        X_test, y_test = test_data[lgb_features],test_data['target']\n",
    "        train_set = lgb.Dataset(X_train, label=y_train,categorical_feature=categorical_features,free_raw_data=False)\n",
    "        test_set = lgb.Dataset(X_test, label=y_test,categorical_feature=categorical_features,free_raw_data=False)\n",
    "\n",
    "    if is_nn:\n",
    "\n",
    "        X_train, y_train    = second_pass_for_nn(train_data,numerical_features,categorical_features)\n",
    "        X_test, y_test      = second_pass_for_nn(test_data,numerical_features,categorical_features)\n",
    "    \n",
    "    if is_rnn:\n",
    "\n",
    "        train_batches, train_targets = second_pass_for_rnn(train_data, rnn_numerical_features, rnn_categorical_features, window_size)\n",
    "        test_batches, test_targets  = second_pass_for_rnn(test_data, rnn_numerical_features, rnn_categorical_features, window_size)\n",
    "        print(f\"train batches shape:{train_batches.shape}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2e496e35",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.270374Z",
     "iopub.status.busy": "2023-12-18T20:16:01.269713Z",
     "iopub.status.idle": "2023-12-18T20:16:01.284346Z",
     "shell.execute_reply": "2023-12-18T20:16:01.283481Z"
    },
    "papermill": {
     "duration": 0.027456,
     "end_time": "2023-12-18T20:16:01.286670",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.259214",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title training models\n",
    "# train_data = flatten_outliers(train_data, 'target', lower_quantile=0.01, upper_quantile=0.99)\n",
    "if train_models:\n",
    "        \n",
    "        directory = os.path.dirname(models_path)\n",
    "        if not os.path.exists(directory):\n",
    "            os.makedirs(directory)\n",
    "\n",
    "        if is_lgb:\n",
    "            lgb_models = []\n",
    "            for i in range(num_models['lgb']):\n",
    "                rnd_state=42+i\n",
    "                print(f\"Training model {i+1} out of {num_models['lgb']} with seed {rnd_state}\")\n",
    "                print(\"---------------------------------------\")\n",
    "\n",
    "                lgb_params['random_state'] = rnd_state\n",
    "                lgb_model = lgb.train(lgb_params, train_set,init_model=None, valid_sets=[train_set, test_set],verbose_eval=50, early_stopping_rounds=250)\n",
    "                lgb_model.save_model(f'{models_path}model_lgb_{i}.txt')\n",
    "                lgb_models.append(lgb_model)\n",
    "                        \n",
    "                if dates_train[1]!=480:\n",
    "                    pred = lgb_model.predict(X_test)\n",
    "                    mae = mean_absolute_error(test_data['target'] , pred)\n",
    "                    print(f\"Mean Absolute Error on test data: {mae:.5f}\")\n",
    "\n",
    "            if dates_train[1]!=480:\n",
    "                predictions =  make_predictions(lgb_models, test_data[lgb_features],model = 'lgb')                   \n",
    "                print(f\"LGB Ensemble Mean Absolute Error: {mean_absolute_error(test_data['target'], predictions):.5f}\")\n",
    "   \n",
    "        if is_nn:\n",
    "\n",
    "            callbacks = [BestScoresCallback()]  # Always include BestScoresCallback\n",
    "            if dates_train[1] != 480:\n",
    "                early_stopping = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=False)\n",
    "                callbacks.append(early_stopping)\n",
    "\n",
    "            if tpu_strategy:\n",
    "                with tpu_strategy.scope():\n",
    "\n",
    "                    nn_models = []\n",
    "                    for i in range(num_models['nn']):\n",
    "                        print(f\"Training nn model {i+1} out of {num_models['nn']} with seed {42+i}\")\n",
    "                        print(\"---------------------------------------\")\n",
    "                        set_all_seeds(42+i)\n",
    "                        \n",
    "                        nn_model = create_model(categorical_features, numerical_features, initial_learning_rate=nn_lr)\n",
    "                        history = nn_model.fit(X_train, y_train, validation_data=(X_test, y_test),epochs=nn_ep, batch_size=nn_bs, callbacks=callbacks)\n",
    "\n",
    "                        print(\"---------------------------------------\")\n",
    "                        nn_model.save(f'{models_path}swish_model_seed_{i}.h5')\n",
    "                        nn_models.append(nn_model)\n",
    "                        \n",
    "                    predictions =  make_predictions(nn_models, X_test,model ='nn')                        \n",
    "\n",
    "                    print(f\"Ensemble Mean Absolute Error: {mean_absolute_error(y_test, predictions):.4f}\")\n",
    "\n",
    "        if is_rnn:\n",
    "\n",
    "            callbacks = [BestScoresCallback()]  # Always include BestScoresCallback\n",
    "            if dates_train[1] != 480:\n",
    "                early_stopping = EarlyStopping(monitor='val_loss', patience=20, restore_best_weights=False)\n",
    "                callbacks.append(early_stopping)\n",
    "\n",
    "            if True or tpu_strategy:\n",
    "                # with tpu_strategy.scope():\n",
    "\n",
    "                    rnn_models = []\n",
    "                    for i in range(num_models['rnn']):\n",
    "\n",
    "                        print(f\"Training rnn model {i+1} out of {num_models['rnn']} with seed {42+i}\")\n",
    "                        print(\"---------------------------------------\")\n",
    "                        set_all_seeds(42+i)\n",
    "\n",
    "                        rnn_model = create_rnn_model_with_residual(window_size, rnn_numerical_features, initial_learning_rate=rnn_lr)\n",
    "                        history = rnn_model.fit(train_batches, train_targets, validation_data=(test_batches, test_targets), epochs=rnn_ep, batch_size=rnn_bs, callbacks=callbacks)\n",
    "                        print(\"---------------------------------------\")\n",
    "                        rnn_model.save(f'{models_path}rnn_model_seed_{i}.h5')\n",
    "                        rnn_models.append(rnn_model)\n",
    "\n",
    "                    predictions =  make_predictions(rnn_models, test_batches,model = 'nn')                        \n",
    "                    print(f\"Ensemble Mean Absolute Error: {mean_absolute_error(test_targets, predictions):.4f}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e1d214df",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.304963Z",
     "iopub.status.busy": "2023-12-18T20:16:01.304233Z",
     "iopub.status.idle": "2023-12-18T20:16:01.309531Z",
     "shell.execute_reply": "2023-12-18T20:16:01.308464Z"
    },
    "papermill": {
     "duration": 0.017118,
     "end_time": "2023-12-18T20:16:01.311840",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.294722",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title comments\n",
    "\n",
    "#=================================================  258 feat  ==============================================\n",
    "\n",
    "#391-480 public-validation seed 42,43: lr:0.05,31 <sub,col 0.2,0.3>  depth = 14\n",
    "#----------- [3281]\ttraining's l1: 6.10386\tvalid_1's l1: 5.86598\n",
    "#----------- [2840]\ttraining's l1: 6.12476\tvalid_1's l1: 5.86619\n",
    "#LGB Ensemble Mean Absolute Error: 5.8637\n",
    "\n",
    "\n",
    "#391-480 public-validation seed 42,43: lr:0.005,511 <sub,col 0.2,0.3>  depth = 14\n",
    "# ----------------- [5670]\ttraining's l1: 5.91513\tvalid_1's l1: 5.86133\n",
    "#------------------ [4606]\ttraining's l1: 5.95538\tvalid_1's l1: 5.8615\n",
    "# LGB Ensemble Mean Absolute Error: 5.8610\n",
    "\n",
    "\n",
    "#391-480 public-validation seed 42,43: lr:0.005,1023 <sub,col 0.2,0.3>  depth = 14\n",
    "# ----------------- [4028]\ttraining's l1: 5.79751\tvalid_1's l1: 5.86096\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#=================================================  253 feat  ==============================================\n",
    "\n",
    "#391-480 public-validation seed 42,43: lr:0.05,31 <sub,col 0.2,0.3>  depth = 14\n",
    "#----------- [3058]\ttraining's l1: 6.11684\tvalid_1's l1: 5.8669\n",
    "#----------- [3453]\ttraining's l1: 6.09743\tvalid_1's l1: 5.86741\n",
    "#LGB Ensemble Mean Absolute Error: 5.8647\n",
    "\n",
    "\n",
    "#391-480 public-validation seed 42,43: lr:0.005,511 <sub,col 0.2,0.3>  depth = 14\n",
    "# ----------------- [5242]\ttraining's l1: 5.92851\tvalid_1's l1: 5.86238\n",
    "#------------------ [6000]\ttraining's l1: 5.90175\tvalid_1's l1: 5.86203\n",
    "\n",
    "#LGB Ensemble Mean Absolute Error: 5.8618\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7404b0b2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.330305Z",
     "iopub.status.busy": "2023-12-18T20:16:01.329584Z",
     "iopub.status.idle": "2023-12-18T20:16:01.335209Z",
     "shell.execute_reply": "2023-12-18T20:16:01.334392Z"
    },
    "papermill": {
     "duration": 0.01738,
     "end_time": "2023-12-18T20:16:01.337285",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.319905",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title importance\n",
    "imp = False\n",
    "if imp:\n",
    "    importances = lgb_model.feature_importance()\n",
    "    feature_names = lgb_model.feature_name()\n",
    "\n",
    "    # Creating a DataFrame for easy manipulation\n",
    "    importance_df = pd.DataFrame({\n",
    "        'Feature': feature_names,\n",
    "        'Importance': importances\n",
    "    })\n",
    "    importance_df.to_csv(\"importance.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "de84bcce",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.355711Z",
     "iopub.status.busy": "2023-12-18T20:16:01.354963Z",
     "iopub.status.idle": "2023-12-18T20:16:01.371257Z",
     "shell.execute_reply": "2023-12-18T20:16:01.370369Z"
    },
    "papermill": {
     "duration": 0.028349,
     "end_time": "2023-12-18T20:16:01.373614",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.345265",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title timeseries functions\n",
    "\n",
    "def clean_format(df):\n",
    "    df['target'] = df['target'].astype('float64')\n",
    "    for feat in ['stock_id','date_id','seconds_in_bucket']:\n",
    "        df[feat] = df[feat].astype('int64')\n",
    "    return df\n",
    "\n",
    "def clean_up(df):\n",
    "    df = df[['stock_id','revealed_date_id','seconds_in_bucket','revealed_target']].rename(columns={'revealed_date_id': 'date_id', 'revealed_target': 'target'})\n",
    "    df['target'].fillna(-0.06020069, inplace=True)\n",
    "    df = clean_format(df)\n",
    "    return df\n",
    "\n",
    "def manage_buffer(buffer, df, max_lag):\n",
    "\n",
    "    # df['iteration'] = df['iteration'].max()  # Ensure the iteration number is consistent in the new df chunk\n",
    "    if buffer.empty:\n",
    "        return df.copy()\n",
    "    elif buffer['iteration'].nunique() < max_lag:\n",
    "        return pd.concat([buffer, df], ignore_index=True)\n",
    "    else:\n",
    "        oldest_iteration = buffer['iteration'].min()\n",
    "        buffer = buffer[buffer['iteration'] > oldest_iteration]\n",
    "        return pd.concat([buffer, df], ignore_index=True)\n",
    "    \n",
    "\n",
    "def append_target_lags(df, target_buffer, lags):\n",
    "\n",
    "    column_to_lag = 'target'\n",
    "    df_lagged = df.copy()\n",
    "\n",
    "    for lag in lags:\n",
    "\n",
    "        temp_df = target_buffer.copy()\n",
    "        temp_df['date_id'] = temp_df['date_id'] + lag\n",
    "        \n",
    "        temp_df.rename(columns={column_to_lag: f'lag{lag}_{column_to_lag}'}, inplace=True)\n",
    "        \n",
    "        df_lagged = pd.merge(df_lagged, temp_df[['stock_id', 'date_id', 'seconds_in_bucket', f'lag{lag}_{column_to_lag}']], \n",
    "                             on=['stock_id', 'date_id', 'seconds_in_bucket'], \n",
    "                             how='left')\n",
    "        \n",
    "    return df_lagged\n",
    "\n",
    "\n",
    "def timeseries_lag_features(data, columns_to_lag, numbers_of_lag):\n",
    "    for col in columns_to_lag:\n",
    "        for lag in numbers_of_lag:\n",
    "            data[f'{col}_lag{lag}'] = data.groupby(['stock_id'])[col].shift(lag)\n",
    "    return data\n",
    "\n",
    "\n",
    "def timeseries_diff_lag_features(data, columns_to_lag, numbers_of_lag):\n",
    "    # Store new columns in a dictionary before joining them to the original DataFrame\n",
    "    new_columns = {}\n",
    "\n",
    "    # Iterate over each group defined by 'stock_id' to reduce groupby operations\n",
    "    grouped_data = data.groupby('stock_id')\n",
    "\n",
    "    for col in columns_to_lag:\n",
    "        for lag in numbers_of_lag:\n",
    "            diff_col_name = f'{col}_diff_lag{lag}'\n",
    "\n",
    "            # Compute the lagged difference for each group\n",
    "            new_columns[diff_col_name] = data[col] - grouped_data[col].shift(lag)\n",
    "\n",
    "    # Concatenate all new columns and join with the original DataFrame\n",
    "    new_columns_df = pd.DataFrame(new_columns)\n",
    "    result_df = pd.concat([data, new_columns_df], axis=1)\n",
    "\n",
    "    return result_df\n",
    "\n",
    "\n",
    "def timeseries_cumsum_features(df, columns_to_compute):\n",
    "    df_copy = df.copy()\n",
    "    \n",
    "    # Group by 'stock_id' and 'date_id' for cumulative sum calculation\n",
    "    grouped = df_copy.groupby(['stock_id'])\n",
    "    \n",
    "    # Calculate cumulative sum for each column within each group\n",
    "    for column in columns_to_compute:\n",
    "        cumsum_col_name = f'{column}_cumsum'\n",
    "        df_copy[cumsum_col_name] = grouped[column].cumsum()\n",
    "        \n",
    "    return df_copy\n",
    "\n",
    "\n",
    "def timeseries_deviation_within_seconds(df, num_features):\n",
    "    # Calculating the median for each numerical feature\n",
    "    medians = df[num_features].median()\n",
    "\n",
    "    # Calculate deviations in a vectorized manner\n",
    "    deviation_cols = {f'deviation_from_median_{feature}': df[feature] - medians[feature] for feature in num_features}\n",
    "    df = df.assign(**deviation_cols)\n",
    "\n",
    "    return df\n",
    "\n",
    "\n",
    "import psutil\n",
    "\n",
    "def memory_limit_exceeded(threshold_in_mb):\n",
    "\n",
    "    process = psutil.Process()\n",
    "    current_memory_usage = process.memory_info().rss / (2**20)  # Convert to MB\n",
    "    return current_memory_usage > threshold_in_mb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b75f50f9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.391372Z",
     "iopub.status.busy": "2023-12-18T20:16:01.390975Z",
     "iopub.status.idle": "2023-12-18T20:16:01.653375Z",
     "shell.execute_reply": "2023-12-18T20:16:01.652209Z"
    },
    "papermill": {
     "duration": 0.273907,
     "end_time": "2023-12-18T20:16:01.655603",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.381696",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if kaggle:\n",
    "    import gc\n",
    "    # remove train_small and train and clear the memory\n",
    "    del train\n",
    "    gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b844a3cf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:16:01.674040Z",
     "iopub.status.busy": "2023-12-18T20:16:01.673675Z",
     "iopub.status.idle": "2023-12-18T20:17:47.710701Z",
     "shell.execute_reply": "2023-12-18T20:17:47.709709Z"
    },
    "papermill": {
     "duration": 106.04916,
     "end_time": "2023-12-18T20:17:47.713152",
     "exception": false,
     "start_time": "2023-12-18T20:16:01.663992",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading lgb models...\n",
      "Done!\n",
      "loading nn models...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator StandardScaler from version 1.3.2 when using version 1.2.2. This might lead to breaking code or invalid results. Use at your own risk. For more info please refer to:\n",
      "https://scikit-learn.org/stable/model_persistence.html#security-maintainability-limitations\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n",
      "loading rnn models...\n",
      "Done!\n",
      "This version of the API is not optimized and should not be used to estimate the runtime of your code on the hidden test set.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:177: UserWarning: Found `n_estimators` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n",
      "  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.029650 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 58215\n",
      "[LightGBM] [Info] Number of data points in the train set: 11000, number of used features: 244\n",
      "[5001]\ttraining's l1: 4.94885\n",
      "[5002]\ttraining's l1: 4.94748\n",
      "[5003]\ttraining's l1: 4.94614\n",
      "[5004]\ttraining's l1: 4.94499\n",
      "[5005]\ttraining's l1: 4.94365\n",
      "[5006]\ttraining's l1: 4.94235\n",
      "[5007]\ttraining's l1: 4.94114\n",
      "[5008]\ttraining's l1: 4.93975\n",
      "[5009]\ttraining's l1: 4.93848\n",
      "[5010]\ttraining's l1: 4.93715\n",
      "[5011]\ttraining's l1: 4.93581\n",
      "[5012]\ttraining's l1: 4.93462\n",
      "[5013]\ttraining's l1: 4.93332\n",
      "[5014]\ttraining's l1: 4.93198\n",
      "[5015]\ttraining's l1: 4.93064\n",
      "[5016]\ttraining's l1: 4.92933\n",
      "[5017]\ttraining's l1: 4.92806\n",
      "[5018]\ttraining's l1: 4.92676\n",
      "[5019]\ttraining's l1: 4.92561\n",
      "[5020]\ttraining's l1: 4.92431\n",
      "Epoch 1/6\n",
      "1/1 [==============================] - 4s 4s/step - loss: 5.1493\n",
      "Epoch 2/6\n",
      "1/1 [==============================] - 1s 654ms/step - loss: 5.1457\n",
      "Epoch 3/6\n",
      "1/1 [==============================] - 1s 631ms/step - loss: 5.1351\n",
      "Epoch 4/6\n",
      "1/1 [==============================] - 1s 650ms/step - loss: 5.1283\n",
      "Epoch 5/6\n",
      "1/1 [==============================] - 1s 634ms/step - loss: 5.1096\n",
      "Epoch 6/6\n",
      "1/1 [==============================] - 1s 650ms/step - loss: 5.1099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:177: UserWarning: Found `n_estimators` in params. Will use it instead of argument\n",
      "  _log_warning(f\"Found `{alias}` in params. Will use it instead of argument\")\n",
      "/opt/conda/lib/python3.10/site-packages/lightgbm/engine.py:239: UserWarning: 'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. Pass 'log_evaluation()' callback via 'callbacks' argument instead.\n",
      "  _log_warning(\"'verbose_eval' argument is deprecated and will be removed in a future release of LightGBM. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] Auto-choosing col-wise multi-threading, the overhead of testing was 0.060239 seconds.\n",
      "You can set `force_col_wise=true` to remove the overhead.\n",
      "[LightGBM] [Info] Total Bins 59235\n",
      "[LightGBM] [Info] Number of data points in the train set: 11000, number of used features: 248\n",
      "[5021]\ttraining's l1: 5.27298\n",
      "[5022]\ttraining's l1: 5.27184\n",
      "[5023]\ttraining's l1: 5.27065\n",
      "[5024]\ttraining's l1: 5.26958\n",
      "[5025]\ttraining's l1: 5.26841\n",
      "[5026]\ttraining's l1: 5.26724\n",
      "[5027]\ttraining's l1: 5.26607\n",
      "[5028]\ttraining's l1: 5.26484\n",
      "[5029]\ttraining's l1: 5.26362\n",
      "[5030]\ttraining's l1: 5.26248\n",
      "[5031]\ttraining's l1: 5.26126\n",
      "[5032]\ttraining's l1: 5.26006\n",
      "[5033]\ttraining's l1: 5.25898\n",
      "[5034]\ttraining's l1: 5.25795\n",
      "[5035]\ttraining's l1: 5.25674\n",
      "[5036]\ttraining's l1: 5.25571\n",
      "[5037]\ttraining's l1: 5.2547\n",
      "[5038]\ttraining's l1: 5.25335\n",
      "[5039]\ttraining's l1: 5.25231\n",
      "[5040]\ttraining's l1: 5.25134\n",
      "Epoch 1/6\n",
      "1/1 [==============================] - 4s 4s/step - loss: 5.5649\n",
      "Epoch 2/6\n",
      "1/1 [==============================] - 1s 642ms/step - loss: 5.5658\n",
      "Epoch 3/6\n",
      "1/1 [==============================] - 1s 636ms/step - loss: 5.5443\n",
      "Epoch 4/6\n",
      "1/1 [==============================] - 1s 620ms/step - loss: 5.5428\n",
      "Epoch 5/6\n",
      "1/1 [==============================] - 1s 627ms/step - loss: 5.5166\n",
      "Epoch 6/6\n",
      "1/1 [==============================] - 1s 617ms/step - loss: 5.5249\n"
     ]
    }
   ],
   "source": [
    "#@title is inference\n",
    "if is_inference:\n",
    "\n",
    "    if online_learning:\n",
    "        \n",
    "        #lgb\n",
    "        lgb_params['learning_rate'] = 0.001 #0.001\n",
    "        lgb_params['n_estimators']  = 20    #10,20\n",
    "        lgb_params['num_leaves']    = 127    #127\n",
    "        lgb_params['device']        = 'cpu'\n",
    "\n",
    "        #nn\n",
    "        online_nn_lr_rate    = 2e-4 #2e-4\n",
    "        online_nn_batch_size = 16384 #16384\n",
    "        online_nn_epochs     = 6     #2,4\n",
    "\n",
    "        #rnn\n",
    "        online_rnn_lr_rate    = 2e-4 #2e-4\n",
    "        online_rnn_batch_size = 2**12 #2**12 \n",
    "        online_rnn_epochs     = 4      #2\n",
    "\n",
    "\n",
    "\n",
    "    if kaggle:\n",
    "        import optiver2023\n",
    "        env = optiver2023.make_env()\n",
    "        iter_test = env.iter_test()\n",
    "    else:\n",
    "        env = MockApi()\n",
    "        iter_test = env.iter_test()\n",
    "    \n",
    "    if load_models:\n",
    "        \n",
    "\n",
    "\n",
    "        if is_lgb:\n",
    "            print(\"loading lgb models...\")\n",
    "            lgb_models = []\n",
    "            for i in range(num_models['lgb']):\n",
    "                loaded_model = lgb.Booster(model_file=f'{models_path}model_lgb_{i}.txt')\n",
    "                lgb_models.append(loaded_model)\n",
    "            \n",
    "            lgb_features = lgb_models[0].feature_name()\n",
    "            print(\"Done!\")\n",
    "\n",
    "\n",
    "        if is_nn:\n",
    "\n",
    "            print(\"loading nn models...\")\n",
    "            loaded_data          = load_pickle(f'{models_path}all_data.pkl')\n",
    "            scaler               = loaded_data[\"scaler\"]\n",
    "            medians              = loaded_data[\"medians\"]\n",
    "            categorical_features = loaded_data[\"categorical_features\"]\n",
    "            numerical_features   = loaded_data[\"numerical_features\"]\n",
    "\n",
    "            nn_models = []\n",
    "            for i in range(num_models['nn']):\n",
    "                loaded_model = load_model(f\"{models_path}swish_model_seed_{i}.h5\")\n",
    "                nn_models.append(loaded_model)\n",
    "            print(\"Done!\")\n",
    "\n",
    "        if is_rnn:\n",
    "\n",
    "            print(\"loading rnn models...\")\n",
    "            loaded_data          = load_pickle(f'{models_path}rnn_all_data.pkl')\n",
    "            rnn_scaler               = loaded_data[\"rnn_scaler\"]\n",
    "            rnn_medians              = loaded_data[\"rnn_medians\"]\n",
    "            rnn_categorical_features = loaded_data[\"rnn_categorical_features\"]\n",
    "            rnn_numerical_features   = loaded_data[\"rnn_numerical_features\"]\n",
    "\n",
    "            rnn_models = []\n",
    "            for i in range(num_models['rnn']):\n",
    "                loaded_model = load_model(f\"{models_path}rnn_model_seed_{i}.h5\")\n",
    "                rnn_models.append(loaded_model)\n",
    "            print(\"Done!\")\n",
    "\n",
    "\n",
    "\n",
    "    #---------------------------buffer management ---------------------------\n",
    "    max_target_lag  = num_of_target_lags\n",
    "    i               = 0\n",
    "    target_counter  = 0\n",
    "    target_buffer   = pd.DataFrame()\n",
    "    total_test      = pd.DataFrame()\n",
    "    daily_online    = pd.DataFrame()\n",
    "    #---------------------------buffer management ---------------------------\n",
    "\n",
    "\n",
    "    for (test, revealed_targets, sample_prediction) in iter_test:\n",
    "        \n",
    "        try:\n",
    "            test = convert_price_cols_float32(test)\n",
    "\n",
    "\n",
    "            if test.seconds_in_bucket.iloc[0] == 0:\n",
    "            \n",
    "                #----------------------------- storing previous targets -------------------------\n",
    "\n",
    "                try:\n",
    "                    revealed_targets = clean_up(revealed_targets)\n",
    "                    revealed_targets['iteration'] = target_counter\n",
    "                    target_buffer = manage_buffer(target_buffer, revealed_targets, max_target_lag)\n",
    "                    target_counter += 1\n",
    "                    \n",
    "                except Exception as e:\n",
    "                    print(f\"An error occurred: {e}\")                \n",
    "\n",
    "\n",
    "                if manage_memory:\n",
    "                    if memory_limit_exceeded(24576):\n",
    "                        is_rnn_online = False\n",
    "                    if memory_limit_exceeded(26276):\n",
    "                        is_nn_online  = False\n",
    "                    if memory_limit_exceeded(28276):\n",
    "                        is_lgb_online = False\n",
    "\n",
    "\n",
    "                try:\n",
    "                    if not daily_online.empty and online_learning:\n",
    "\n",
    "                        daily_online.drop(columns='target', inplace=True)\n",
    "                        daily_online = pd.merge(daily_online, revealed_targets[['stock_id', 'date_id', 'seconds_in_bucket', 'target']], \n",
    "                                                    on=['stock_id', 'date_id', 'seconds_in_bucket'], \n",
    "                                                    how='inner')\n",
    "\n",
    "\n",
    "                        if is_lgb and is_lgb_online:\n",
    "                            online_lgbs = []\n",
    "\n",
    "                            for lgb_model in lgb_models:\n",
    "                                new_data = lgb.Dataset(daily_online[lgb_features], label=daily_online['target'])\n",
    "                                online_lgb = lgb.train(lgb_params, new_data, init_model=lgb_model,valid_sets=[new_data,new_data],keep_training_booster=True,verbose_eval=1)\n",
    "                                online_lgbs.append(online_lgb)\n",
    "\n",
    "                            #update the lgb_models\n",
    "                            lgb_models = online_lgbs.copy()\n",
    "\n",
    "                        if is_nn and is_nn_online:\n",
    "\n",
    "                            X_online, y_online          = second_pass_for_nn(daily_online,numerical_features,categorical_features,is_inference=True)\n",
    "\n",
    "                            for nn_model in nn_models:\n",
    "                                lr_schedule = ExponentialDecay(initial_learning_rate=online_nn_lr_rate,decay_steps=10000,decay_rate=0.96,staircase=True)\n",
    "                                optimizer = Adam(learning_rate=lr_schedule)\n",
    "                                nn_model.compile(optimizer=optimizer, loss = \"mean_absolute_error\")\n",
    "                                nn_model.fit(X_online, y_online, epochs= online_nn_epochs, batch_size=online_nn_batch_size) \n",
    "                                K.clear_session()\n",
    "                                gc.collect()  \n",
    "\n",
    "                        if is_rnn and is_rnn_online:\n",
    "\n",
    "                            online_batches, online_targets = online_pass_for_rnn(daily_online, rnn_numerical_features, rnn_categorical_features, window_size)\n",
    "                            \n",
    "                            for rnn_model in rnn_models:\n",
    "                                lr_schedule = ExponentialDecay( initial_learning_rate=online_rnn_lr_rate, decay_steps=10000, decay_rate=0.8,staircase=True)\n",
    "                                optimizer = Adam(learning_rate=lr_schedule)\n",
    "                                rnn_model.compile(optimizer=optimizer, loss = \"mean_absolute_error\")\n",
    "                                rnn_model.fit(online_batches, online_targets, epochs=online_rnn_epochs, batch_size=online_rnn_batch_size)\n",
    "                                K.clear_session()\n",
    "                                gc.collect()  \n",
    "\n",
    "                except Exception as e:\n",
    "                    print(f\"An error occurred: {e}\")\n",
    "\n",
    "                daily_online = pd.DataFrame()\n",
    "                #----------------------------- storing previous targets -------------------------\n",
    "\n",
    "\n",
    "            test['target'] = 0\n",
    "\n",
    "            #--------------- connected ------------\n",
    "            test = feature_engineering(test)\n",
    "            test = compute_imbalances(test, columns_sizes,prefix='_sz_')\n",
    "            test = compute_imbalances(test, columns_prices,prefix = '_pr_')\n",
    "\n",
    "            test = append_target_lags(test, target_buffer, target_lags)\n",
    "            \n",
    "            test = calculate_stat_lag(test, num_lags=num_of_target_lags)\n",
    "\n",
    "            eng_features       = [feature for feature in test.columns if \"_eng\" in feature]\n",
    "            imb_features_all   = [feature for feature in test.columns if \"_imb_\" in feature]\n",
    "            imb_features_price = [feature for feature in test.columns if \"_pr_\" in feature]\n",
    "            imb_features_size  = [feature for feature in test.columns if \"_sz_\" in feature]\n",
    "            #--------------- connected ------------\n",
    "\n",
    "            #------------------------------------  this for deviation within seconds  -------------------------\n",
    "            deviation_cols = raw_cols + eng_features + imb_features_size + imb_features_price\n",
    "            test = timeseries_deviation_within_seconds(test,deviation_cols)\n",
    "\n",
    "            test = map_global(test,aggregated_dic)\n",
    "\n",
    "            #------------------------ get the full data up to current seconds in bucket in date-----------------------------------\n",
    "            if not daily_online.empty:  \n",
    "                full_data = pd.concat([daily_online[test.columns], test], ignore_index=True)\n",
    "            else:\n",
    "                full_data = test\n",
    "\n",
    "\n",
    "            diff_lag_cols = raw_cols + eng_features \n",
    "            full_data = timeseries_diff_lag_features(full_data, diff_lag_cols, diff_lags)\n",
    "\n",
    "            cumsum_columns = columns_sizes + imb_features_size + eng_features  \n",
    "            full_data = timeseries_cumsum_features(full_data, cumsum_columns)\n",
    "\n",
    "        #------------------------ get the full data up to current seconds in bucket in date-----------------------------------\n",
    "\n",
    "\n",
    "            # getting back to test size\n",
    "            test = full_data.iloc[-test.shape[0]:].copy()\n",
    "        \n",
    "        \n",
    "            #------------------------------------  Global based on stock_id ---------------------------------------\n",
    "            test.replace([np.inf, -np.inf], np.nan, inplace=True)\n",
    "            \n",
    "\n",
    "            if test.currently_scored.iloc[0]:\n",
    "                if is_lgb:\n",
    "                    test['target'] = ens_models[0]*make_predictions(lgb_models,test[lgb_features],model='lgb')\n",
    "\n",
    "                if is_nn:\n",
    "                    X_test, _           = second_pass_for_nn(test,numerical_features,categorical_features,is_inference=True)\n",
    "                    test['target'] += ens_models[1]*make_predictions(nn_models, X_test,model='nn').flatten()\n",
    "\n",
    "                if is_rnn:\n",
    "                    X_test, _           = second_pass_for_rnn(full_data, rnn_numerical_features, rnn_categorical_features, window_size, is_inference=True)\n",
    "                    test['target'] += ens_models[2]*make_predictions(rnn_models, X_test,model='nn').flatten()\n",
    "\n",
    "\n",
    "\n",
    "            if not kaggle:\n",
    "                total_test = pd.concat([total_test, test], ignore_index=True)\n",
    "\n",
    "            \n",
    "            daily_online = pd.concat([daily_online, test], ignore_index=True)\n",
    "\n",
    "\n",
    "            sample_prediction = pd.merge(sample_prediction.drop(columns='target'), test[['row_id', 'target']], on=['row_id'], how='left')\n",
    "            sample_prediction['target'].fillna(0, inplace=True)\n",
    "            sample_prediction['target'].replace([np.inf, -np.inf], 0, inplace=True)\n",
    "\n",
    "        except Exception as e:\n",
    "            sample_prediction['target'] = 0\n",
    "\n",
    "        env.predict(sample_prediction)\n",
    "        i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "0c5e1a9c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:17:47.741820Z",
     "iopub.status.busy": "2023-12-18T20:17:47.741028Z",
     "iopub.status.idle": "2023-12-18T20:17:47.749816Z",
     "shell.execute_reply": "2023-12-18T20:17:47.748737Z"
    },
    "papermill": {
     "duration": 0.025333,
     "end_time": "2023-12-18T20:17:47.752083",
     "exception": false,
     "start_time": "2023-12-18T20:17:47.726750",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title check inference\n",
    "\n",
    "if  is_inference and not kaggle: \n",
    "\n",
    "    last_date = True\n",
    "    if last_date:\n",
    "        original = test_data.query(\"date_id >= 480\").reset_index()\n",
    "        loop_test = total_test.query(\"date_id >= 480\").reset_index()\n",
    "    else:\n",
    "        original = test_data\n",
    "        loop_test = total_test\n",
    "\n",
    "\n",
    "    if is_lgb:\n",
    "        original['preds'] =  make_predictions(lgb_models, original[features],model='lgb')\n",
    "        mae = mean_absolute_error(original['target'], original['preds'])\n",
    "        print(f\"Mean Absolute Error lgb: {mae:.4f}\")\n",
    "\n",
    "    if is_nn:\n",
    "        X_test, y_test  = second_pass_for_nn(original,numerical_features,categorical_features)\n",
    "        predictions =  make_predictions(nn_models, X_test,model='nn')\n",
    "        print(f\"Mean Absolute Error nn: {mean_absolute_error(y_test, predictions):.4f}\")\n",
    "    \n",
    "    if is_rnn:\n",
    "        X_test, y_test  = second_pass_for_rnn(original,rnn_numerical_features,rnn_categorical_features,window_size)\n",
    "        predictions =  make_predictions(rnn_models, X_test,model='nn')\n",
    "        print(f\"Mean Absolute Error rnn: {mean_absolute_error(y_test, predictions):.4f}\")\n",
    "\n",
    "    mae = mean_absolute_error(original['target'], loop_test['target'])\n",
    "    print(f\"Mean Absolute Error on loop inference : {mae:.4f}\")\n",
    "    \n",
    "\n",
    "#0.05, 31 full vaidation Mean Absolute Error lgb on origin data 480: 4.7701\n",
    "# # test discrepencies\n",
    "# common_columns = loop_test.columns.intersection(original.columns)\n",
    "\n",
    "# # Step 2: Reorder columns in both DataFrames\n",
    "# loop_test_common = loop_test[common_columns]\n",
    "# original_common = original[common_columns]\n",
    "# loop_test_common.describe().to_csv('loop.csv')\n",
    "# original_common.describe().to_csv(\"original.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "86737719",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:17:47.780335Z",
     "iopub.status.busy": "2023-12-18T20:17:47.779963Z",
     "iopub.status.idle": "2023-12-18T20:17:47.794568Z",
     "shell.execute_reply": "2023-12-18T20:17:47.793671Z"
    },
    "papermill": {
     "duration": 0.031447,
     "end_time": "2023-12-18T20:17:47.796585",
     "exception": false,
     "start_time": "2023-12-18T20:17:47.765138",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title all preds \n",
    "\n",
    "\n",
    "def assign_predictions_and_reorder(test_data, preds):\n",
    "\n",
    "\n",
    "    test_data['original_order'] = test_data.index\n",
    "    test_data_sorted = test_data.sort_values(by=['stock_id', 'date_id'])\n",
    "    test_data_sorted['rnn_preds'] = preds\n",
    "    test_data_original_order = test_data_sorted.sort_values(by='original_order')\n",
    "    return test_data_original_order['rnn_preds'].values\n",
    "\n",
    "\n",
    "def second_pass_for_rnn_sim(df, rnn_numerical_features, rnn_categorical_features, window_size):\n",
    "    # Check if the DataFrame is empty\n",
    "    global rnn_scaler,rnn_medians\n",
    "    \n",
    "    if df.empty:\n",
    "        return None, None\n",
    "\n",
    "    # Work on a copy of the DataFrame to avoid changing the original df\n",
    "    df_copy = df.copy()\n",
    "\n",
    "    # Standard scaling for numerical features\n",
    "    df_copy.fillna(rnn_medians, inplace=True)\n",
    "    df_copy[rnn_numerical_features] = rnn_scaler.transform(df_copy[rnn_numerical_features])\n",
    "        \n",
    "    # Preprocess Data\n",
    "    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n",
    "    df_copy['imbalance_buy_sell_flag'] += 1 \n",
    "\n",
    "    df_copy_batches, df_copy_targets = create_batches(df_copy, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "\n",
    "    df_copy_batches = np.array(df_copy_batches)\n",
    "    df_copy_targets = np.array(df_copy_targets)\n",
    "\n",
    "    return df_copy_batches, df_copy_targets\n",
    "\n",
    "\n",
    "if colab:\n",
    "    simulation_path = \"/content/drive/My Drive/optiver/optiver-258-rnn-vals-74/\"\n",
    "else:\n",
    "    simulation_path = \"vals-258/\"\n",
    "\n",
    "\n",
    "simulation_ens = False\n",
    "if simulation_ens:\n",
    "\n",
    "    models_path = simulation_path\n",
    "\n",
    "    lgb_model             = lgb.Booster(model_file=f'{models_path}model_lgb_0.txt')\n",
    "    lgb_features          = lgb_model.feature_name()\n",
    "    lgb_preds             = lgb_model.predict(test_data[lgb_features])\n",
    "\n",
    "\n",
    "    loaded_data          = load_pickle(f'{models_path}all_data.pkl')\n",
    "    scaler               = loaded_data[\"scaler\"]\n",
    "    medians              = loaded_data[\"medians\"]\n",
    "    categorical_features = loaded_data[\"categorical_features\"]\n",
    "    numerical_features   = loaded_data[\"numerical_features\"]\n",
    "    nn_model             = load_model(f\"{models_path}swish_model_seed_0.h5\")\n",
    "    X_test, _            = second_pass_for_nn(test_data,numerical_features,categorical_features,is_inference = True)\n",
    "    nn_preds             = nn_model.predict(X_test, batch_size=16384).flatten()\n",
    "\n",
    "\n",
    "\n",
    "    loaded_data                 = load_pickle(f'{models_path}rnn_all_data.pkl')\n",
    "    rnn_scaler                  = loaded_data[\"rnn_scaler\"]\n",
    "    rnn_medians                 = loaded_data[\"rnn_medians\"]\n",
    "    rnn_categorical_features    = loaded_data[\"rnn_categorical_features\"]\n",
    "    rnn_numerical_features      = loaded_data[\"rnn_numerical_features\"]\n",
    "    rnn_model                   = load_model(f\"{models_path}rnn_model_seed_0.h5\")\n",
    "    X_test, _                   = second_pass_for_rnn_sim(test_data,rnn_numerical_features,rnn_categorical_features,window_size)\n",
    "    rnn_preds                   = rnn_model.predict(X_test, batch_size=16384).flatten()\n",
    "    rnn_preds                   = assign_predictions_and_reorder(test_data,rnn_preds)\n",
    "\n",
    "\n",
    "\n",
    "    test_data['lgb_preds']  = lgb_preds\n",
    "    test_data['nn_preds']   = nn_preds\n",
    "    test_data['rnn_preds']  = rnn_preds\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "45ed163b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:17:47.824983Z",
     "iopub.status.busy": "2023-12-18T20:17:47.824532Z",
     "iopub.status.idle": "2023-12-18T20:17:47.836911Z",
     "shell.execute_reply": "2023-12-18T20:17:47.835873Z"
    },
    "papermill": {
     "duration": 0.028755,
     "end_time": "2023-12-18T20:17:47.838965",
     "exception": false,
     "start_time": "2023-12-18T20:17:47.810210",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "if simulation_ens:\n",
    "    mae_scores = {}\n",
    "\n",
    "    # List of model predictions\n",
    "    model_predictions = ['lgb_preds','nn_preds' ,'rnn_preds']\n",
    "\n",
    "    # Calculate MAE for each model\n",
    "    for model_pred in model_predictions:\n",
    "        mae = mean_absolute_error(test_data['target'], test_data[model_pred])\n",
    "        mae_scores[model_pred] = mae\n",
    "        print(f\"{model_pred} Score on Test: {mae}\")\n",
    "    \n",
    "    def simple_average_ensemble(test_data):\n",
    "        test_data['ensemble_pred'] = (test_data['lgb_preds'] + test_data['nn_preds'] + test_data['rnn_preds']) / 3\n",
    "        return test_data\n",
    "    \n",
    "    def weighted_average_ensemble(test_data, weights):\n",
    "\n",
    "        total_weight = sum(weights.values())\n",
    "        test_data['ensemble_pred'] = (test_data['lgb_preds'] * weights['lgb'] +  test_data['nn_preds'] * weights['nn'] +  test_data['rnn_preds'] * weights['rnn']) / total_weight\n",
    "        return test_data\n",
    "\n",
    "\n",
    "    test_data = simple_average_ensemble(test_data)\n",
    "\n",
    "    avg_score = mean_absolute_error(test_data['target'], test_data['ensemble_pred'])\n",
    "\n",
    "    print(f\"average Score on Test: {avg_score}\")\n",
    "    \n",
    "    weights = {'lgb': 0.5, 'nn': 0.25, 'rnn': 0.25}  \n",
    "    test_data = weighted_average_ensemble(test_data, weights)\n",
    "    avg_score = mean_absolute_error(test_data['target'], test_data['ensemble_pred'])\n",
    "\n",
    "    print(f\"weighted average Score on Test: {avg_score}\")\n",
    "\n",
    "    hm= (2*(test_data['nn_preds']*test_data['rnn_preds']))/(test_data['nn_preds']+test_data['rnn_preds'])\n",
    "\n",
    "    avg_score = mean_absolute_error(test_data['target'],.5*test_data['lgb_preds']  + .5*hm )\n",
    "\n",
    "    print(f\"harmonic mean weighted average Score on Test: {avg_score}\")\n",
    "\n",
    "\n",
    "    print(\"---------------------------\")\n",
    "\n",
    "\n",
    "    test_data_418 =  test_data.query(f\"date_id >= 418\").reset_index().copy()\n",
    "\n",
    "    test_data_000 =  test_data.query(f\"date_id < 418\").reset_index().copy()\n",
    "\n",
    "    for model_pred in model_predictions:\n",
    "        mae = mean_absolute_error(test_data_418['target'], test_data_418[model_pred])\n",
    "        mae_scores[model_pred] = mae\n",
    "        print(f\"{model_pred} Score on Test 418: {mae}\")\n",
    "\n",
    "\n",
    "    test_data_418 = simple_average_ensemble(test_data_418)\n",
    "\n",
    "    avg_score = mean_absolute_error(test_data_418['target'], test_data_418['ensemble_pred'])\n",
    "\n",
    "    print(f\"average Score on Test: {avg_score}\")\n",
    "    \n",
    "    weights = {'lgb': 0.5, 'nn': 0.25, 'rnn': 0.25}  \n",
    "    test_data_418 = weighted_average_ensemble(test_data_418, weights)\n",
    "    avg_score = mean_absolute_error(test_data_418['target'], test_data_418['ensemble_pred'])\n",
    "\n",
    "    print(f\"weighted average Score on Test 418: {avg_score}\")\n",
    "\n",
    "    print(\"---------------------------\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "547d52cc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:17:47.867194Z",
     "iopub.status.busy": "2023-12-18T20:17:47.866395Z",
     "iopub.status.idle": "2023-12-18T20:17:47.871178Z",
     "shell.execute_reply": "2023-12-18T20:17:47.870173Z"
    },
    "papermill": {
     "duration": 0.021108,
     "end_time": "2023-12-18T20:17:47.873291",
     "exception": false,
     "start_time": "2023-12-18T20:17:47.852183",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# lgb_preds Score on Test: 5.860964545705968\n",
    "# nn_preds Score on Test: 5.871083373009854\n",
    "# rnn_preds Score on Test: 5.8716475533769525\n",
    "# average Score on Test: 5.8557724356023275\n",
    "# weighted average Score on Test: 5.854561274377742\n",
    "# ---------------------------\n",
    "# lgb_preds Score on Test 418: 5.659749933905271\n",
    "# nn_preds Score on Test 418: 5.669579630390044\n",
    "# rnn_preds Score on Test 418: 5.670666008919795\n",
    "# average Score on Test: 5.654617576426593\n",
    "# weighted average Score on Test: 5.6533975842069975"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "1c930089",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2023-12-18T20:17:47.901501Z",
     "iopub.status.busy": "2023-12-18T20:17:47.901091Z",
     "iopub.status.idle": "2023-12-18T20:17:47.950807Z",
     "shell.execute_reply": "2023-12-18T20:17:47.949780Z"
    },
    "papermill": {
     "duration": 0.066826,
     "end_time": "2023-12-18T20:17:47.953148",
     "exception": false,
     "start_time": "2023-12-18T20:17:47.886322",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#@title simulation\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "import math\n",
    "\n",
    "\n",
    "def online_pass_for_rnn_sim(df, rnn_numerical_features, rnn_categorical_features, window_size):\n",
    "    # Check if the DataFrame is empty\n",
    "    global rnn_scaler,rnn_medians\n",
    "    \n",
    "    if df.empty:\n",
    "        return None, None\n",
    "\n",
    "    # Work on a copy of the DataFrame to avoid changing the original df\n",
    "    df_copy = df.copy()\n",
    "\n",
    "    # Standard scaling for numerical features\n",
    "    df_copy.fillna(rnn_medians, inplace=True)\n",
    "    df_copy[rnn_numerical_features] = rnn_scaler.transform(df_copy[rnn_numerical_features])\n",
    "\n",
    "    # Preprocess Data\n",
    "    df_copy['seconds_in_bucket'] = df_copy['seconds_in_bucket'] / 10\n",
    "    df_copy['imbalance_buy_sell_flag'] += 1 \n",
    "\n",
    "\n",
    "    grouped = df_copy.groupby(['stock_id'])\n",
    "    all_batches = []\n",
    "    all_targets = []\n",
    "\n",
    "    for _, group in tqdm(grouped, desc=\"Processing groups\"):\n",
    "        # Precompute sequences for the current group\n",
    "        precomputed_data = precompute_sequences(group, window_size, rnn_numerical_features, rnn_categorical_features)\n",
    "\n",
    "        # Initialize containers for group sequences and targets\n",
    "        group_sequences = []\n",
    "        group_targets = []\n",
    "\n",
    "        # Iterate over the time steps and retrieve precomputed sequences\n",
    "        for time_step in range(55):\n",
    "            sequence, target = get_sequence(precomputed_data, time_step)\n",
    "            if sequence.size > 0: \n",
    "                group_sequences.append(sequence)\n",
    "                group_targets.append(target)\n",
    "\n",
    "        # Extend the main batches with the group's sequences and targets\n",
    "        all_batches.extend(group_sequences)\n",
    "        all_targets.extend(group_targets)\n",
    "\n",
    "    df_batches = np.array(all_batches)\n",
    "    df_targets = np.array(all_targets)\n",
    "\n",
    "\n",
    "    return df_batches, df_targets\n",
    "\n",
    "if simulation:\n",
    "    \n",
    "    models_path = simulation_path\n",
    "    online_learning= True\n",
    "\n",
    "    #-------------------LGB----------------\n",
    "    initial_learning_rates = [0.001]\n",
    "    n_estimators_values    = [20]#,30,40]\n",
    "    num_leaves_values      = [1023]\n",
    "    max_depth_values       = [2]#,4,6,8,10,12,14]\n",
    "    #-------------------LGB----------------\n",
    "\n",
    "    if is_lgb:\n",
    "        print(\"loading lgb models...\")\n",
    "        lgb_models_original = []\n",
    "        for i in range(1):\n",
    "            loaded_model = lgb.Booster(model_file=f'{models_path}model_lgb_{i}.txt')\n",
    "            lgb_models_original.append(loaded_model)\n",
    "\n",
    "        lgb_features = lgb_models_original[0].feature_name()\n",
    "        print(\"making prediction on the whole test data....\")\n",
    "        preds = make_predictions(lgb_models_original,test_data[lgb_features],model='lgb')\n",
    "        mae_original = mean_absolute_error(test_data['target'],preds)\n",
    "        print(f\"lgb Score on Test: {mae_original}\")\n",
    "\n",
    "\n",
    "    #-------------------NN----------------\n",
    "    nn_init_rate_values = [1e-4]#,2e-4,1e-5,5e-4]\n",
    "    online_batch_size_values =[16384]\n",
    "    online_epochs_values = [2]#4,8,10]\n",
    "    #-------------------NN----------------\n",
    "\n",
    "    if is_nn:\n",
    "        print(\"loading nn models...\")\n",
    "        loaded_data          = load_pickle(f'{models_path}all_data.pkl')\n",
    "        scaler               = loaded_data[\"scaler\"]\n",
    "        medians              = loaded_data[\"medians\"]\n",
    "        categorical_features = loaded_data[\"categorical_features\"]\n",
    "        numerical_features   = loaded_data[\"numerical_features\"]\n",
    "        nn_models_original = []\n",
    "        for i in range(1):\n",
    "            loaded_model_original = load_model(f\"{models_path}swish_model_seed_{i}.h5\")\n",
    "            nn_models_original.append(loaded_model_original)\n",
    "        print(\"Done!\")\n",
    "\n",
    "        print(\"making prediction on the whole test data....\")\n",
    "        X_test, _       = second_pass_for_nn(test_data,numerical_features,categorical_features,is_inference=True)\n",
    "        preds           = make_predictions(nn_models_original, X_test,model='nn').flatten()\n",
    "        mae_original    = mean_absolute_error(test_data['target'],preds)\n",
    "        print(f\"nn Score on Test: {mae_original}\")\n",
    "\n",
    "\n",
    "    #-------------------RNN----------------\n",
    "    rnn_init_rate_values = [2e-4,4e-4]#,2e-4]\n",
    "    rnn_online_batch_size_values =[2**12,2**10]#,2**11,2**12,2*9]\n",
    "    rnn_online_epochs_values = [4,6,10]\n",
    "    #-------------------RNN----------------\n",
    "\n",
    "    if is_rnn:\n",
    "        print(\"loading rnn models...\")\n",
    "        loaded_data          = load_pickle(f'{models_path}rnn_all_data.pkl')\n",
    "        rnn_scaler               = loaded_data[\"rnn_scaler\"]\n",
    "        rnn_medians              = loaded_data[\"rnn_medians\"]\n",
    "        rnn_categorical_features = loaded_data[\"rnn_categorical_features\"]\n",
    "        rnn_numerical_features   = loaded_data[\"rnn_numerical_features\"]\n",
    "        rnn_models_original = []\n",
    "        for i in range(1):\n",
    "            loaded_model_original = load_model(f\"{models_path}rnn_model_seed_{i}.h5\")\n",
    "            rnn_models_original.append(loaded_model_original)\n",
    "        print(\"Done!\")\n",
    "\n",
    "        X_test_batches, y_test_batches           = second_pass_for_rnn_sim(test_data,rnn_numerical_features,rnn_categorical_features,window_size)\n",
    "        preds = make_predictions(rnn_models_original, X_test_batches,model='nn').flatten()\n",
    "        preds = assign_predictions_and_reorder(test_data,preds)\n",
    "        mae_original    = mean_absolute_error(test_data['target'],preds)\n",
    "        print(f\"rnn Score on Test: {mae_original}\")\n",
    "\n",
    "\n",
    "\n",
    "    br_count = 200\n",
    "\n",
    "    results = []\n",
    "\n",
    "\n",
    "# Nested loop over all parameter combinations\n",
    "    \n",
    "    for init_lr in initial_learning_rates:\n",
    "        for n_estimators in n_estimators_values:\n",
    "            for num_leaves in num_leaves_values:\n",
    "                    for max_depth in max_depth_values:\n",
    "\n",
    "                        for nn_init_rate in nn_init_rate_values:\n",
    "                            for online_batch_size in online_batch_size_values:\n",
    "                                for online_epochs in online_epochs_values:\n",
    "                                    for rnn_init_rate in rnn_init_rate_values:\n",
    "                                        for rnn_online_batch_size in rnn_online_batch_size_values:\n",
    "                                            for rnn_online_epochs in rnn_online_epochs_values:\n",
    "\n",
    "                                                if is_nn:\n",
    "                                                    print(\"loading nn models...\")\n",
    "                                                    loaded_data          = load_pickle(f'{models_path}all_data.pkl')\n",
    "                                                    scaler               = loaded_data[\"scaler\"]\n",
    "                                                    medians              = loaded_data[\"medians\"]\n",
    "                                                    categorical_features = loaded_data[\"categorical_features\"]\n",
    "                                                    numerical_features   = loaded_data[\"numerical_features\"]\n",
    "\n",
    "                                                    nn_models = []\n",
    "                                                    for i in range(1):\n",
    "                                                        loaded_model = load_model(f\"{models_path}swish_model_seed_{i}.h5\")\n",
    "                                                        nn_models.append(loaded_model)\n",
    "                                                    print(\"Done!\")\n",
    "\n",
    "                                                    nn_models_original = []\n",
    "                                                    for i in range(1):\n",
    "                                                        loaded_model_original = load_model(f\"{models_path}swish_model_seed_{i}.h5\")\n",
    "                                                        nn_models_original.append(loaded_model_original)\n",
    "                                                    print(\"Done!\")\n",
    "\n",
    "                                                if is_rnn:\n",
    "                                                    print(\"loading rnn models...\")\n",
    "                                                    loaded_data          = load_pickle(f'{models_path}rnn_all_data.pkl')\n",
    "                                                    rnn_scaler               = loaded_data[\"rnn_scaler\"]\n",
    "                                                    rnn_medians              = loaded_data[\"rnn_medians\"]\n",
    "                                                    rnn_categorical_features = loaded_data[\"rnn_categorical_features\"]\n",
    "                                                    rnn_numerical_features   = loaded_data[\"rnn_numerical_features\"]\n",
    "\n",
    "                                                    rnn_models = []\n",
    "                                                    for i in range(1):\n",
    "                                                        loaded_model = load_model(f\"{models_path}rnn_model_seed_{i}.h5\")\n",
    "                                                        rnn_models.append(loaded_model)\n",
    "                                                    print(\"Done!\")\n",
    "\n",
    "                                                    rnn_models_original = []\n",
    "                                                    for i in range(1):\n",
    "                                                        loaded_model_original = load_model(f\"{models_path}rnn_model_seed_{i}.h5\")\n",
    "                                                        rnn_models_original.append(loaded_model_original)\n",
    "                                                    print(\"Done!\")\n",
    "\n",
    "                                                if is_lgb:  \n",
    "                                                    print(\"loading lgb models...\")\n",
    "                                                    lgb_models_original = []\n",
    "                                                    for i in range(1):\n",
    "                                                        loaded_model = lgb.Booster(model_file=f'{models_path}model_lgb_{i}.txt')\n",
    "                                                        lgb_models_original.append(loaded_model)\n",
    "\n",
    "\n",
    "                                                    online_models = []\n",
    "                                                    for i in range(1):\n",
    "                                                        loaded_model = lgb.Booster(model_file=f'{models_path}model_lgb_{i}.txt')\n",
    "                                                        online_models.append(loaded_model)\n",
    "                                                    \n",
    "                                                    lgb_features = online_models[0].feature_name()\n",
    "\n",
    "\n",
    "                                                # Set the parameters for your model\n",
    "                                                lgb_params['n_estimators'] = n_estimators\n",
    "                                                lgb_params['num_leaves'] = num_leaves\n",
    "                                                lgb_params['max_depth'] = max_depth\n",
    "                                                lgb_params['learning_rate'] = init_lr\n",
    "\n",
    "                                                mae_values = [] \n",
    "                                                loop_test = pd.DataFrame()\n",
    "                                                subset_cum = pd.DataFrame()\n",
    "                                                counter = 0\n",
    "\n",
    "                                                for date_id in tqdm(test_data['date_id'].unique(), desc=\"Iterating over unique date_ids\"):\n",
    "                                                    \n",
    "\n",
    "                                                    counter +=1\n",
    "                                                    #-------------------updating lr decay-------------------\n",
    "                                                    subset_test = test_data[test_data['date_id'] == date_id]\n",
    "                                                    \n",
    "                                                    if is_lgb:\n",
    "                                                        subset_test['online_preds']   = make_predictions(online_models,subset_test[lgb_features],model='lgb')\n",
    "                                                        subset_test['original_preds'] = make_predictions(lgb_models_original,subset_test[lgb_features],model='lgb')\n",
    "\n",
    "                                                    if is_nn:\n",
    "                                                        X_test, _           = second_pass_for_nn(subset_test,numerical_features,categorical_features,is_inference = True)\n",
    "                                                        subset_test['online_preds']  = make_predictions(nn_models, X_test,model='nn').flatten()\n",
    "                                                        subset_test['original_preds'] = make_predictions(nn_models_original, X_test,model='nn').flatten()\n",
    "\n",
    "                                                    if is_rnn:\n",
    "                                                        X_test_batches, y_test_batches           = online_pass_for_rnn_sim(subset_test,rnn_numerical_features,rnn_categorical_features,window_size)\n",
    "                                                        \n",
    "                                                        online_preds  = make_predictions(rnn_models, X_test_batches,model='nn').flatten()\n",
    "                                                        original_preds = make_predictions(rnn_models_original, X_test_batches,model='nn').flatten()\n",
    "\n",
    "                                                        subset_test['online_preds'] = assign_predictions_and_reorder(subset_test,online_preds)\n",
    "                                                        subset_test['original_preds'] = assign_predictions_and_reorder(subset_test,original_preds)\n",
    "\n",
    "                                                \n",
    "                                                    mae_online = mean_absolute_error(subset_test['target'], subset_test['online_preds'])\n",
    "                                                    mae_original = mean_absolute_error(subset_test['target'], subset_test['original_preds'])\n",
    "\n",
    "                                                    print(f\"-------------------------------------------- Online: Mean Absolute Error lgb: {mae_online:.5f}\")\n",
    "                                                    print(f\"-------------------------------------------original: Mean Absolute Error lgb: {mae_original:.5f}\")\n",
    "\n",
    "\n",
    "                                                    mae_values.append({'mae_online': mae_online, 'mae_original': mae_original})\n",
    "\n",
    "                                                    loop_test = pd.concat([loop_test, subset_test], ignore_index=True)\n",
    "                                                    subset_cum = pd.concat([subset_cum, subset_test], ignore_index=True)\n",
    "\n",
    "                                                    if counter >=br_count:\n",
    "                                                        break\n",
    "\n",
    "                                                    if online_learning and subset_cum.date_id.nunique() >= 1:\n",
    "\n",
    "                                                        if is_lgb:\n",
    "                                                            print(f\"==================learning rate = {lgb_params['learning_rate']}==================\")\n",
    "                                                            #----------------------------------------   LGB online learning   ----------------------------------------\n",
    "                                                            temp_lgbs = []\n",
    "                                                            for lgb_model in online_models:\n",
    "                                                                new_data = lgb.Dataset(subset_test[lgb_features], label=subset_test['target'],free_raw_data=False)\n",
    "                                                                temp_lgb = lgb.train(lgb_params, new_data, init_model=lgb_model,valid_sets=[new_data,new_data],keep_training_booster=True,verbose_eval=5)\n",
    "                                                                temp_lgbs.append(temp_lgb)\n",
    "\n",
    "                                                        \n",
    "\n",
    "                                                            online_models = temp_lgbs.copy()\n",
    "                                                            #----------------------------------------   LGB online learning   ----------------------------------------\n",
    "                                                        \n",
    "                                                        if is_nn:\n",
    "\n",
    "                                                            print(\"updating NN model....\")\n",
    "                                                            for nn_model in nn_models:\n",
    "\n",
    "                                                                # for layer in nn_model.layers[:-5]:\n",
    "                                                                #     layer.trainable = False\n",
    "                                                            \n",
    "                                                                lr_schedule = ExponentialDecay( initial_learning_rate=nn_init_rate, decay_steps=10000, decay_rate=0.9,staircase=True)\n",
    "                                                                optimizer = Adam(learning_rate=lr_schedule)\n",
    "                                                                nn_model.compile(optimizer=optimizer, loss = \"mean_absolute_error\")\n",
    "                                                                X_online, y_online    = second_pass_for_nn(subset_test,numerical_features,categorical_features,is_inference=True)\n",
    "                                                                history = nn_model.fit(X_online, y_online, epochs=online_epochs, batch_size=online_batch_size) \n",
    "\n",
    "                                                        if is_rnn:\n",
    "\n",
    "                                                            online_batches, online_targets = online_pass_for_rnn_sim(subset_test, rnn_numerical_features, rnn_categorical_features, window_size)\n",
    "                    \n",
    "                                                            for rnn_model in rnn_models:\n",
    "                                                                lr_schedule = ExponentialDecay( initial_learning_rate=rnn_init_rate, decay_steps=10000, decay_rate=0.8,staircase=True)\n",
    "                                                                optimizer = Adam(learning_rate=lr_schedule)\n",
    "                                                                rnn_model.compile(optimizer=optimizer, loss = \"mean_absolute_error\")\n",
    "                                                                rnn_model.fit(online_batches, online_targets, epochs=rnn_online_epochs, batch_size=rnn_online_batch_size)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "                                                        subset_cum  = pd.DataFrame()\n",
    "\n",
    "\n",
    "                                                mae_online = mean_absolute_error(loop_test['target'], loop_test['online_preds'])\n",
    "                                                mae_original = mean_absolute_error(loop_test['target'], loop_test['original_preds'])\n",
    "\n",
    "                                                print(f\"Total Online: Mean Absolute Error lgb: {mae_online:.5f}\")\n",
    "                                                print(f\"Total Original: Mean Absolute Error lgb: {mae_original:.5f}\")\n",
    "\n",
    "\n",
    "                                                loop_test_418 =  loop_test.query(f\"date_id >= 418\").reset_index().copy()\n",
    "                                                mae_online_418 = mean_absolute_error(loop_test_418['target'], loop_test_418['online_preds'])\n",
    "                                                mae_original_418 = mean_absolute_error(loop_test_418['target'], loop_test_418['original_preds'])\n",
    "\n",
    "                                                results.append({\n",
    "                                                    'initial_learning_rate': init_lr,\n",
    "                                                    'n_estimators': n_estimators,\n",
    "                                                    'num_leaves': num_leaves,\n",
    "                                                    'max_depth':max_depth,\n",
    "\n",
    "                                                    'nn_init_rate':nn_init_rate,\n",
    "                                                    'online_batch_size':online_batch_size,\n",
    "                                                    'online_epochs':online_epochs,\n",
    "\n",
    "                                                    'rnn_init_rate':rnn_init_rate,\n",
    "                                                    'rnn_online_batch_size':rnn_online_batch_size,\n",
    "                                                    'rnn_online_epochs':rnn_online_epochs,\n",
    "\n",
    "                                                    'mae_online': mae_online,\n",
    "                                                    'mae_original': mae_original,\n",
    "                                                    'mae_online_418_end': mae_online_418,\n",
    "                                                    'mae_original_418_end': mae_original_418\n",
    "                                                })\n",
    "                                                pd.DataFrame(results).to_csv(f\"{models_path}iter_rnn_result_online.csv\", index=False)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "    results_df = pd.DataFrame(results)\n",
    "    results_df.to_csv(f\"{models_path}final_result_rnn_online.csv\")\n",
    "\n",
    "\n",
    "    loop_test_418 =  loop_test.query(f\"date_id >= 418\").reset_index().copy()\n",
    "    mae_online_418 = mean_absolute_error(loop_test_418['target'], loop_test_418['online_preds'])\n",
    "    mae_original_418 = mean_absolute_error(loop_test_418['target'], loop_test_418['original_preds'])\n",
    "\n",
    "    print(f\"Total Online: Mean Absolute Error lgb on 418-end : {mae_online_418:.5f}\")\n",
    "    print(f\"Total Original: Mean Absolute Error lgb on 418-end: {mae_original_418:.5f}\")\n",
    "\n",
    "# Total Online: Mean Absolute Error lgb on 418-end : 5.65682\n",
    "# Total Original: Mean Absolute Error lgb on 418-end: 5.65876\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6678fb0f",
   "metadata": {
    "papermill": {
     "duration": 0.012712,
     "end_time": "2023-12-18T20:17:47.979664",
     "exception": false,
     "start_time": "2023-12-18T20:17:47.966952",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "databundleVersionId": 7056235,
     "sourceId": 57891,
     "sourceType": "competition"
    },
    {
     "datasetId": 4165853,
     "sourceId": 7201705,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30626,
   "isGpuEnabled": false,
   "isInternetEnabled": false,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 152.364413,
   "end_time": "2023-12-18T20:17:50.652848",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2023-12-18T20:15:18.288435",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
